func (this *uint32Writer) WriteMsg(msg proto.Message) (err error) {
var data []byte
if m, ok := msg.(marshaler); ok {
n := m.Size()
if n >= len(this.buffer) {
this.buffer = make([]byte, n)
}
_, err = m.MarshalTo(this.buffer)
if err != nil {
return err
}
data = this.buffer[:n]
} else {
data, err = proto.Marshal(msg)
if err != nil {
return err
}
}
length := uint32(len(data))
if err := binary.Write(this.w, this.byteOrder, &length); err != nil {
return err
}
_, err = this.w.Write(data)
return err
}
func TestGetExtensionStability(t *testing.T) {
check := func(m *pb.MyMessage) bool {
ext1, err := proto.GetExtension(m, pb.E_Ext_More)
if err != nil {
t.Fatalf("GetExtension() failed: %s", err)
}
ext2, err := proto.GetExtension(m, pb.E_Ext_More)
if err != nil {
t.Fatalf("GetExtension() failed: %s", err)
}
return ext1 == ext2
}
msg := &pb.MyMessage{Count: proto.Int32(4)}
ext0 := &pb.Ext{}
if err := proto.SetExtension(msg, pb.E_Ext_More, ext0); err != nil {
t.Fatalf("Could not set ext1: %s", ext0)
}
if !check(msg) {
t.Errorf("GetExtension() not stable before marshaling")
}
bb, err := proto.Marshal(msg)
if err != nil {
t.Fatalf("Marshal() failed: %s", err)
}
msg1 := &pb.MyMessage{}
err = proto.Unmarshal(bb, msg1)
if err != nil {
t.Fatalf("Unmarshal() failed: %s", err)
}
if !check(msg1) {
t.Errorf("GetExtension() not stable after unmarshaling")
}
}
func (this *varintWriter) WriteMsg(msg proto.Message) (err error) {
var data []byte
if m, ok := msg.(marshaler); ok {
n := m.Size()
if n >= len(this.buffer) {
this.buffer = make([]byte, n)
}
_, err = m.MarshalTo(this.buffer)
if err != nil {
return err
}
data = this.buffer[:n]
} else {
data, err = proto.Marshal(msg)
if err != nil {
return err
}
}
length := uint64(len(data))
n := binary.PutUvarint(this.lenBuf, length)
_, err = this.w.Write(this.lenBuf[:n])
if err != nil {
return err
}
_, err = this.w.Write(data)
return err
}
// MarshalPrivateKey converts a key object into its protobuf serialized form.
func MarshalPrivateKey(k PrivKey) ([]byte, error) {
b := MarshalRsaPrivateKey(k.(*RsaPrivateKey))
pmes := new(pb.PrivateKey)
typ := pb.KeyType_RSA // for now only type.
pmes.Type = &typ
pmes.Data = b
return proto.Marshal(pmes)
}
// putLocal stores the key value pair in the datastore
func (dht *IpfsDHT) putLocal(key key.Key, rec *pb.Record) error {
data, err := proto.Marshal(rec)
if err != nil {
return err
}
return dht.datastore.Put(key.DsKey(), data)
}
func (n *FSNode) GetBytes() ([]byte, error) {
pbn := new(pb.Data)
pbn.Type = &n.Type
pbn.Filesize = proto.Uint64(uint64(len(n.Data)) + n.subtotal)
pbn.Blocksizes = n.blocksizes
pbn.Data = n.Data
return proto.Marshal(pbn)
}
func (sk *RsaPrivateKey) Bytes() ([]byte, error) {
b := x509.MarshalPKCS1PrivateKey(sk.sk)
pbmes := new(pb.PrivateKey)
typ := pb.KeyType_RSA
pbmes.Type = &typ
pbmes.Data = b
return proto.Marshal(pbmes)
}
func newTestMessage() *pb.MyMessage {
msg := &pb.MyMessage{
Count: proto.Int32(42),
Name: proto.String("Dave"),
Quote: proto.String(`"I didn't want to go."`),
Pet: []string{"bunny", "kitty", "horsey"},
Inner: &pb.InnerMessage{
Host: proto.String("footrest.syd"),
Port: proto.Int32(7001),
Connected: proto.Bool(true),
},
Others: []*pb.OtherMessage{
{
Key: proto.Int64(0xdeadbeef),
Value: []byte{1, 65, 7, 12},
},
{
Weight: proto.Float32(6.022),
Inner: &pb.InnerMessage{
Host: proto.String("lesha.mtv"),
Port: proto.Int32(8002),
},
},
},
Bikeshed: pb.MyMessage_BLUE.Enum(),
Somegroup: &pb.MyMessage_SomeGroup{
GroupField: proto.Int32(8),
},
// One normally wouldn't do this.
// This is an undeclared tag 13, as a varint (wire type 0) with value 4.
XXX_unrecognized: []byte{13<<3 | 0, 4},
}
ext := &pb.Ext{
Data: proto.String("Big gobs for big rats"),
}
if err := proto.SetExtension(msg, pb.E_Ext_More, ext); err != nil {
panic(err)
}
greetings := []string{"adg", "easy", "cow"}
if err := proto.SetExtension(msg, pb.E_Greeting, greetings); err != nil {
panic(err)
}
// Add an unknown extension. We marshal a pb.Ext, and fake the ID.
b, err := proto.Marshal(&pb.Ext{Data: proto.String("3G skiing")})
if err != nil {
panic(err)
}
b = append(proto.EncodeVarint(201<<3|proto.WireBytes), b...)
proto.SetRawExtension(msg, 201, b)
// Extensions can be plain fields, too, so let's test that.
b = append(proto.EncodeVarint(202<<3|proto.WireVarint), 19)
proto.SetRawExtension(msg, 202, b)
return msg
}
func writeHdr(n *merkledag.Node, hdr *pb.Set) error {
hdrData, err := proto.Marshal(hdr)
if err != nil {
return err
}
n.Data = make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData))
written := binary.PutUvarint(n.Data, uint64(len(hdrData)))
n.Data = n.Data[:written]
n.Data = append(n.Data, hdrData...)
return nil
}
// MarshalPublicKey converts a public key object into a protobuf serialized
// public key
func MarshalPublicKey(k PubKey) ([]byte, error) {
b, err := MarshalRsaPublicKey(k.(*RsaPublicKey))
if err != nil {
return nil, err
}
pmes := new(pb.PublicKey)
typ := pb.KeyType_RSA // for now only type.
pmes.Type = &typ
pmes.Data = b
return proto.Marshal(pmes)
}
func (pk *RsaPublicKey) Bytes() ([]byte, error) {
b, err := x509.MarshalPKIXPublicKey(pk.k)
if err != nil {
return nil, err
}
pbmes := new(pb.PublicKey)
typ := pb.KeyType_RSA
pbmes.Type = &typ
pbmes.Data = b
return proto.Marshal(pbmes)
}
func putRoutingRecord(ds datastore.Datastore, k key.Key, value *dhtpb.Record) error {
data, err := proto.Marshal(value)
if err != nil {
return err
}
dskey := k.DsKey()
// TODO namespace
if err := ds.Put(dskey, data); err != nil {
return err
}
return nil
}
// Returns Bytes that represent a Directory
func FolderPBData() []byte {
pbfile := new(pb.Data)
typ := pb.Data_Directory
pbfile.Type = &typ
data, err := proto.Marshal(pbfile)
if err != nil {
//this really shouldnt happen, i promise
panic(err)
}
return data
}
func (c *offlineRouting) PutValue(ctx context.Context, key key.Key, val []byte) error {
rec, err := record.MakePutRecord(c.sk, key, val, false)
if err != nil {
return err
}
data, err := proto.Marshal(rec)
if err != nil {
return err
}
return c.datastore.Put(key.DsKey(), data)
}
func SymlinkData(path string) ([]byte, error) {
pbdata := new(pb.Data)
typ := pb.Data_Symlink
pbdata.Data = []byte(path)
pbdata.Type = &typ
out, err := proto.Marshal(pbdata)
if err != nil {
return nil, err
}
return out, nil
}
func BytesForMetadata(m *Metadata) ([]byte, error) {
pbd := new(pb.Data)
pbd.Filesize = proto.Uint64(m.Size)
typ := pb.Data_Metadata
pbd.Type = &typ
mdd, err := m.Bytes()
if err != nil {
return nil, err
}
pbd.Data = mdd
return proto.Marshal(pbd)
}
// FIXME(brian): is this method meant to simulate putting a value into the network?
func (c *client) PutValue(ctx context.Context, key key.Key, val []byte) error {
log.Debugf("PutValue: %s", key)
rec := new(dhtpb.Record)
rec.Value = val
rec.Key = proto.String(string(key))
rec.TimeReceived = proto.String(u.FormatRFC3339(time.Now()))
data, err := proto.Marshal(rec)
if err != nil {
return err
}
return c.datastore.Put(key.DsKey(), data)
}
func WrapData(b []byte) []byte {
pbdata := new(pb.Data)
typ := pb.Data_Raw
pbdata.Data = b
pbdata.Type = &typ
out, err := proto.Marshal(pbdata)
if err != nil {
// This shouldnt happen. seriously.
panic(err)
}
return out
}
func createRoutingEntryData(pk ci.PrivKey, val path.Path) ([]byte, error) {
entry := new(pb.IpnsEntry)
entry.Value = []byte(val)
typ := pb.IpnsEntry_EOL
entry.ValidityType = &typ
entry.Validity = []byte(u.FormatRFC3339(time.Now().Add(time.Hour * 24)))
sig, err := pk.Sign(ipnsEntryDataForSig(entry))
if err != nil {
return nil, err
}
entry.Signature = sig
return proto.Marshal(entry)
}
func TestPBNodeVerboseEqual(t *testing.T) {
popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
p := NewPopulatedPBNode(popr, false)
data, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
panic(err)
}
msg := &PBNode{}
if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil {
panic(err)
}
if err := p.VerboseEqual(msg); err != nil {
t.Fatalf("%#v !VerboseEqual %#v, since %v", msg, p, err)
}
}
func PublishEntry(ctx context.Context, r routing.IpfsRouting, ipnskey key.Key, rec *pb.IpnsEntry) error {
timectx, cancel := context.WithTimeout(ctx, PublishPutValTimeout)
defer cancel()
data, err := proto.Marshal(rec)
if err != nil {
return err
}
log.Debugf("Storing ipns entry at: %s", ipnskey)
// Store ipns entry at "/ipns/"+b58(h(pubkey))
if err := r.PutValue(timectx, ipnskey, data); err != nil {
return err
}
return nil
}
func BenchmarkPBLinkProtoMarshal(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
pops := make([]*PBLink, 10000)
for i := 0; i < 10000; i++ {
pops[i] = NewPopulatedPBLink(popr, false)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
data, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000])
if err != nil {
panic(err)
}
total += len(data)
}
b.SetBytes(int64(total / b.N))
}
func FilePBData(data []byte, totalsize uint64) []byte {
pbfile := new(pb.Data)
typ := pb.Data_File
pbfile.Type = &typ
pbfile.Data = data
pbfile.Filesize = proto.Uint64(totalsize)
data, err := proto.Marshal(pbfile)
if err != nil {
// This really shouldnt happen, i promise
// The only failure case for marshal is if required fields
// are not filled out, and they all are. If the proto object
// gets changed and nobody updates this function, the code
// should panic due to programmer error
panic(err)
}
return data
}
// Store a value in this peer local storage
func (dht *IpfsDHT) handlePutValue(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
defer log.EventBegin(ctx, "handlePutValue", p).Done()
dskey := key.Key(pmes.GetKey()).DsKey()
if err := dht.verifyRecordLocally(pmes.GetRecord()); err != nil {
log.Debugf("Bad dht record in PUT from: %s. %s", key.Key(pmes.GetRecord().GetAuthor()), err)
return nil, err
}
data, err := proto.Marshal(pmes.GetRecord())
if err != nil {
return nil, err
}
err = dht.datastore.Put(dskey, data)
log.Debugf("%s handlePutValue %v", dht.self, dskey)
return pmes, err
}
func TestPBNodeSize(t *testing.T) {
popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
p := NewPopulatedPBNode(popr, true)
size2 := github_com_gogo_protobuf_proto.Size(p)
data, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
panic(err)
}
size := p.Size()
if len(data) != size {
t.Fatalf("size %v != marshalled size %v", size, len(data))
}
if size2 != size {
t.Fatalf("size %v != before marshal proto.Size %v", size, size2)
}
size3 := github_com_gogo_protobuf_proto.Size(p)
if size3 != size {
t.Fatalf("size %v != after marshal proto.Size %v", size, size3)
}
}
func TestPBLinkProto(t *testing.T) {
popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
p := NewPopulatedPBLink(popr, false)
data, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
panic(err)
}
msg := &PBLink{}
if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil {
panic(err)
}
for i := range data {
data[i] = byte(popr.Intn(256))
}
if err := p.VerboseEqual(msg); err != nil {
t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err)
}
if !p.Equal(msg) {
t.Fatalf("%#v !Proto %#v", msg, p)
}
}
func (this *fullWriter) WriteMsg(msg proto.Message) (err error) {
var data []byte
if m, ok := msg.(marshaler); ok {
n := m.Size()
if n >= len(this.buffer) {
this.buffer = make([]byte, n)
}
_, err = m.MarshalTo(this.buffer)
if err != nil {
return err
}
data = this.buffer[:n]
} else {
data, err = proto.Marshal(msg)
if err != nil {
return err
}
}
_, err = this.w.Write(data)
return err
}
func BenchmarkPBLinkProtoUnmarshal(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
datas := make([][]byte, 10000)
for i := 0; i < 10000; i++ {
data, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedPBLink(popr, false))
if err != nil {
panic(err)
}
datas[i] = data
}
msg := &PBLink{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
total += len(datas[i%10000])
if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil {
panic(err)
}
}
b.SetBytes(int64(total / b.N))
}
func AssertSelected(r *pb.IpnsEntry, from ...*pb.IpnsEntry) error {
shuffle(from)
var vals [][]byte
for _, r := range from {
data, err := proto.Marshal(r)
if err != nil {
return err
}
vals = append(vals, data)
}
i, err := selectRecord(from, vals)
if err != nil {
return err
}
if from[i] != r {
return fmt.Errorf("selected incorrect record %d", i)
}
return nil
}
// Store a value in this peer local storage
func (dht *IpfsDHT) handlePutValue(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
defer log.EventBegin(ctx, "handlePutValue", p).Done()
dskey := key.Key(pmes.GetKey()).DsKey()
if err := dht.verifyRecordLocally(pmes.GetRecord()); err != nil {
log.Warningf("Bad dht record in PUT from: %s. %s", key.Key(pmes.GetRecord().GetAuthor()), err)
return nil, err
}
rec := pmes.GetRecord()
// record the time we receive every record
rec.TimeReceived = proto.String(u.FormatRFC3339(time.Now()))
data, err := proto.Marshal(rec)
if err != nil {
return nil, err
}
err = dht.datastore.Put(dskey, data)
log.Debugf("%s handlePutValue %v", dht.self, dskey)
return pmes, err
}