func TestPBLinkMarshalTo(t *testing.T) {
popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
p := NewPopulatedPBLink(popr, false)
size := p.Size()
data := make([]byte, size)
for i := range data {
data[i] = byte(popr.Intn(256))
}
_, err := p.MarshalTo(data)
if err != nil {
panic(err)
}
msg := &PBLink{}
if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil {
panic(err)
}
for i := range data {
data[i] = byte(popr.Intn(256))
}
if err := p.VerboseEqual(msg); err != nil {
t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err)
}
if !p.Equal(msg) {
t.Fatalf("%#v !Proto %#v", msg, p)
}
}
func TestFSNode(t *testing.T) {
fsn := new(FSNode)
fsn.Type = TFile
for i := 0; i < 15; i++ {
fsn.AddBlockSize(100)
}
fsn.Data = make([]byte, 128)
b, err := fsn.GetBytes()
if err != nil {
t.Fatal(err)
}
pbn := new(pb.Data)
err = proto.Unmarshal(b, pbn)
if err != nil {
t.Fatal(err)
}
ds, err := DataSize(b)
if err != nil {
t.Fatal(err)
}
if ds != (100*15)+128 {
t.Fatal("Datasize calculations incorrect!")
}
}
func readMsgCtx(ctx context.Context, r msgio.Reader, p proto.Message) ([]byte, error) {
var msg []byte
// read in a goroutine so we can exit when our context is cancelled.
done := make(chan error)
go func() {
var err error
msg, err = r.ReadMsg()
select {
case done <- err:
case <-ctx.Done():
}
}()
select {
case <-ctx.Done():
return nil, ctx.Err()
case e := <-done:
if e != nil {
return nil, e
}
}
return msg, proto.Unmarshal(msg, p)
}
// NewDagReader creates a new reader object that reads the data represented by the given
// node, using the passed in DAGService for data retreival
func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (*DagReader, error) {
pb := new(ftpb.Data)
if err := proto.Unmarshal(n.Data, pb); err != nil {
return nil, err
}
switch pb.GetType() {
case ftpb.Data_Directory:
// Dont allow reading directories
return nil, ErrIsDir
case ftpb.Data_Raw:
fallthrough
case ftpb.Data_File:
return NewDataFileReader(ctx, n, pb, serv), nil
case ftpb.Data_Metadata:
if len(n.Links) == 0 {
return nil, errors.New("incorrectly formatted metadata object")
}
child, err := n.Links[0].GetNode(ctx, serv)
if err != nil {
return nil, err
}
return NewDagReader(ctx, child, serv)
case ftpb.Data_Symlink:
return nil, ErrCantReadSymlinks
default:
return nil, ft.ErrUnrecognizedType
}
}
func TestGetExtensionStability(t *testing.T) {
check := func(m *pb.MyMessage) bool {
ext1, err := proto.GetExtension(m, pb.E_Ext_More)
if err != nil {
t.Fatalf("GetExtension() failed: %s", err)
}
ext2, err := proto.GetExtension(m, pb.E_Ext_More)
if err != nil {
t.Fatalf("GetExtension() failed: %s", err)
}
return ext1 == ext2
}
msg := &pb.MyMessage{Count: proto.Int32(4)}
ext0 := &pb.Ext{}
if err := proto.SetExtension(msg, pb.E_Ext_More, ext0); err != nil {
t.Fatalf("Could not set ext1: %s", ext0)
}
if !check(msg) {
t.Errorf("GetExtension() not stable before marshaling")
}
bb, err := proto.Marshal(msg)
if err != nil {
t.Fatalf("Marshal() failed: %s", err)
}
msg1 := &pb.MyMessage{}
err = proto.Unmarshal(bb, msg1)
if err != nil {
t.Fatalf("Unmarshal() failed: %s", err)
}
if !check(msg1) {
t.Errorf("GetExtension() not stable after unmarshaling")
}
}
// getLocal attempts to retrieve the value from the datastore
func (dht *IpfsDHT) getLocal(key key.Key) ([]byte, error) {
log.Debug("getLocal %s", key)
v, err := dht.datastore.Get(key.DsKey())
if err != nil {
return nil, err
}
log.Debug("found in db")
byt, ok := v.([]byte)
if !ok {
return nil, errors.New("value stored in datastore not []byte")
}
rec := new(pb.Record)
err = proto.Unmarshal(byt, rec)
if err != nil {
return nil, err
}
// TODO: 'if paranoid'
if u.Debug {
err = dht.verifyRecordLocally(rec)
if err != nil {
log.Debugf("local record verify failed: %s (discarded)", err)
return nil, err
}
}
return rec.GetValue(), nil
}
func (this *fullReader) ReadMsg(msg proto.Message) error {
length, err := this.r.Read(this.buf)
if err != nil {
return err
}
return proto.Unmarshal(this.buf[:length], msg)
}
func UnwrapData(data []byte) ([]byte, error) {
pbdata := new(pb.Data)
err := proto.Unmarshal(data, pbdata)
if err != nil {
return nil, err
}
return pbdata.GetData(), nil
}
func FromBytes(data []byte) (*pb.Data, error) {
pbdata := new(pb.Data)
err := proto.Unmarshal(data, pbdata)
if err != nil {
return nil, err
}
return pbdata, nil
}
func MetadataFromBytes(b []byte) (*Metadata, error) {
pbd := new(pb.Data)
err := proto.Unmarshal(b, pbd)
if err != nil {
return nil, err
}
if pbd.GetType() != pb.Data_Metadata {
return nil, errors.New("incorrect node type")
}
pbm := new(pb.Metadata)
err = proto.Unmarshal(pbd.Data, pbm)
if err != nil {
return nil, err
}
md := new(Metadata)
md.MimeType = pbm.GetMimeType()
return md, nil
}
// resolveOnce implements resolver. Uses the IPFS routing system to
// resolve SFS-like names.
func (r *routingResolver) resolveOnce(ctx context.Context, name string) (path.Path, error) {
log.Debugf("RoutingResolve: '%s'", name)
hash, err := mh.FromB58String(name)
if err != nil {
log.Warning("RoutingResolve: bad input hash: [%s]\n", name)
return "", err
}
// name should be a multihash. if it isn't, error out here.
// use the routing system to get the name.
// /ipns/<name>
h := []byte("/ipns/" + string(hash))
ipnsKey := key.Key(h)
val, err := r.routing.GetValue(ctx, ipnsKey)
if err != nil {
log.Warning("RoutingResolve get failed.")
return "", err
}
entry := new(pb.IpnsEntry)
err = proto.Unmarshal(val, entry)
if err != nil {
return "", err
}
// name should be a public key retrievable from ipfs
pubkey, err := routing.GetPublicKey(r.routing, ctx, hash)
if err != nil {
return "", err
}
hsh, _ := pubkey.Hash()
log.Debugf("pk hash = %s", key.Key(hsh))
// check sig with pk
if ok, err := pubkey.Verify(ipnsEntryDataForSig(entry), entry.GetSignature()); err != nil || !ok {
return "", fmt.Errorf("Invalid value. Not signed by PrivateKey corresponding to %v", pubkey)
}
// ok sig checks out. this is a valid name.
// check for old style record:
valh, err := mh.Cast(entry.GetValue())
if err != nil {
// Not a multihash, probably a new record
return path.ParsePath(string(entry.GetValue()))
} else {
// Its an old style multihash record
log.Warning("Detected old style multihash record")
return path.FromKey(key.Key(valh)), nil
}
}
func FSNodeFromBytes(b []byte) (*FSNode, error) {
pbn := new(pb.Data)
err := proto.Unmarshal(b, pbn)
if err != nil {
return nil, err
}
n := new(FSNode)
n.Data = pbn.Data
n.blocksizes = pbn.Blocksizes
n.subtotal = pbn.GetFilesize() - uint64(len(n.Data))
n.Type = pbn.GetType()
return n, nil
}
// UnmarshalPrivateKey converts a protobuf serialized private key into its
// representative object
func UnmarshalPrivateKey(data []byte) (PrivKey, error) {
pmes := new(pb.PrivateKey)
err := proto.Unmarshal(data, pmes)
if err != nil {
return nil, err
}
switch pmes.GetType() {
case pb.KeyType_RSA:
return UnmarshalRsaPrivateKey(pmes.GetData())
default:
return nil, ErrBadKeyType
}
}
func TestPBNodeVerboseEqual(t *testing.T) {
popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
p := NewPopulatedPBNode(popr, false)
data, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
panic(err)
}
msg := &PBNode{}
if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil {
panic(err)
}
if err := p.VerboseEqual(msg); err != nil {
t.Fatalf("%#v !VerboseEqual %#v, since %v", msg, p, err)
}
}
func getRoutingProviders(ds datastore.Datastore, k key.Key) ([]*dhtpb.Message_Peer, error) {
e := log.EventBegin(context.Background(), "getProviders", &k)
defer e.Done()
var providers []*dhtpb.Message_Peer
if v, err := ds.Get(providerKey(k)); err == nil {
if data, ok := v.([]byte); ok {
var msg dhtpb.Message
if err := proto.Unmarshal(data, &msg); err != nil {
return nil, err
}
providers = append(providers, msg.GetProviderPeers()...)
}
}
return providers, nil
}
func getRoutingRecord(ds datastore.Datastore, k key.Key) (*dhtpb.Record, error) {
dskey := k.DsKey()
val, err := ds.Get(dskey)
if err != nil {
return nil, err
}
recordBytes, ok := val.([]byte)
if !ok {
return nil, fmt.Errorf("datastore had non byte-slice value for %v", dskey)
}
var record dhtpb.Record
if err := proto.Unmarshal(recordBytes, &record); err != nil {
return nil, errors.New("failed to unmarshal dht record from datastore")
}
return &record, nil
}
func (this *varintReader) ReadMsg(msg proto.Message) error {
length64, err := binary.ReadUvarint(this.r)
if err != nil {
return err
}
length := int(length64)
if length < 0 || length > this.maxSize {
return io.ErrShortBuffer
}
if len(this.buf) < length {
this.buf = make([]byte, length)
}
buf := this.buf[:length]
if _, err := io.ReadFull(this.r, buf); err != nil {
return err
}
return proto.Unmarshal(buf, msg)
}
func (c *offlineRouting) GetValue(ctx context.Context, key key.Key) ([]byte, error) {
v, err := c.datastore.Get(key.DsKey())
if err != nil {
return nil, err
}
byt, ok := v.([]byte)
if !ok {
return nil, errors.New("value stored in datastore not []byte")
}
rec := new(pb.Record)
err = proto.Unmarshal(byt, rec)
if err != nil {
return nil, err
}
return rec.GetValue(), nil
}
func DataSize(data []byte) (uint64, error) {
pbdata := new(pb.Data)
err := proto.Unmarshal(data, pbdata)
if err != nil {
return 0, err
}
switch pbdata.GetType() {
case pb.Data_Directory:
return 0, errors.New("Cant get data size of directory!")
case pb.Data_File:
return pbdata.GetFilesize(), nil
case pb.Data_Raw:
return uint64(len(pbdata.GetData())), nil
default:
return 0, errors.New("Unrecognized node data type!")
}
}
func BenchmarkPBLinkProtoUnmarshal(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
datas := make([][]byte, 10000)
for i := 0; i < 10000; i++ {
data, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedPBLink(popr, false))
if err != nil {
panic(err)
}
datas[i] = data
}
msg := &PBLink{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
total += len(datas[i%10000])
if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil {
panic(err)
}
}
b.SetBytes(int64(total / b.N))
}
func (w *Writer) WriteNode(nd *mdag.Node, fpath string) error {
pb := new(upb.Data)
if err := proto.Unmarshal(nd.Data, pb); err != nil {
return err
}
switch pb.GetType() {
case upb.Data_Metadata:
fallthrough
case upb.Data_Directory:
return w.writeDir(nd, fpath)
case upb.Data_Raw:
fallthrough
case upb.Data_File:
return w.writeFile(nd, pb, fpath)
case upb.Data_Symlink:
return writeSymlinkHeader(w.TarW, string(pb.GetData()), fpath)
default:
return ft.ErrUnrecognizedType
}
}
// ValidateIpnsRecord implements ValidatorFunc and verifies that the
// given 'val' is an IpnsEntry and that that entry is valid.
func ValidateIpnsRecord(k key.Key, val []byte) error {
entry := new(pb.IpnsEntry)
err := proto.Unmarshal(val, entry)
if err != nil {
return err
}
switch entry.GetValidityType() {
case pb.IpnsEntry_EOL:
t, err := u.ParseRFC3339(string(entry.GetValidity()))
if err != nil {
log.Debug("Failed parsing time for ipns record EOL")
return err
}
if time.Now().After(t) {
return ErrExpiredRecord
}
default:
return ErrUnrecognizedValidity
}
return nil
}
// precalcNextBuf follows the next link in line and loads it from the DAGService,
// setting the next buffer to read from
func (dr *DagReader) precalcNextBuf(ctx context.Context) error {
dr.buf.Close() // Just to make sure
if dr.linkPosition >= len(dr.promises) {
return io.EOF
}
nxt, err := dr.promises[dr.linkPosition].Get(ctx)
if err != nil {
return err
}
dr.linkPosition++
pb := new(ftpb.Data)
err = proto.Unmarshal(nxt.Data, pb)
if err != nil {
return fmt.Errorf("incorrectly formatted protobuf: %s", err)
}
switch pb.GetType() {
case ftpb.Data_Directory:
// A directory should not exist within a file
return ft.ErrInvalidDirLocation
case ftpb.Data_File:
dr.buf = NewDataFileReader(dr.ctx, nxt, pb, dr.serv)
return nil
case ftpb.Data_Raw:
dr.buf = NewRSNCFromBytes(pb.GetData())
return nil
case ftpb.Data_Metadata:
return errors.New("Shouldnt have had metadata object inside file")
case ftpb.Data_Symlink:
return errors.New("shouldnt have had symlink inside file")
default:
return ft.ErrUnrecognizedType
}
}
func (s *Node) loadData() error {
s.cached = new(ftpb.Data)
return proto.Unmarshal(s.Nd.Data, s.cached)
}
func (dht *IpfsDHT) handleGetValue(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
defer log.EventBegin(ctx, "handleGetValue", p).Done()
log.Debugf("%s handleGetValue for key: %s", dht.self, pmes.GetKey())
// setup response
resp := pb.NewMessage(pmes.GetType(), pmes.GetKey(), pmes.GetClusterLevel())
// first, is there even a key?
k := pmes.GetKey()
if k == "" {
return nil, errors.New("handleGetValue but no key was provided")
// TODO: send back an error response? could be bad, but the other node's hanging.
}
// let's first check if we have the value locally.
log.Debugf("%s handleGetValue looking into ds", dht.self)
dskey := key.Key(k).DsKey()
iVal, err := dht.datastore.Get(dskey)
log.Debugf("%s handleGetValue looking into ds GOT %v", dht.self, iVal)
// if we got an unexpected error, bail.
if err != nil && err != ds.ErrNotFound {
return nil, err
}
// if we have the value, send it back
if err == nil {
log.Debugf("%s handleGetValue success!", dht.self)
byts, ok := iVal.([]byte)
if !ok {
return nil, fmt.Errorf("datastore had non byte-slice value for %v", dskey)
}
rec := new(pb.Record)
err := proto.Unmarshal(byts, rec)
if err != nil {
log.Debug("Failed to unmarshal dht record from datastore")
return nil, err
}
resp.Record = rec
}
// Find closest peer on given cluster to desired key and reply with that info
closer := dht.betterPeersToQuery(pmes, p, CloserPeerCount)
if len(closer) > 0 {
closerinfos := peer.PeerInfos(dht.peerstore, closer)
for _, pi := range closerinfos {
log.Debugf("handleGetValue returning closer peer: '%s'", pi.ID)
if len(pi.Addrs) < 1 {
log.Errorf(`no addresses on peer being sent!
[local:%s]
[sending:%s]
[remote:%s]`, dht.self, pi.ID, p)
}
}
resp.CloserPeers = pb.PeerInfosToPBPeers(dht.host.Network(), closerinfos)
}
return resp, nil
}