func (this *varintWriter) WriteMsg(msg proto.Message) (err error) {
var data []byte
if m, ok := msg.(marshaler); ok {
n := m.Size()
if n >= len(this.buffer) {
this.buffer = make([]byte, n)
}
_, err = m.MarshalTo(this.buffer)
if err != nil {
return err
}
data = this.buffer[:n]
} else {
data, err = proto.Marshal(msg)
if err != nil {
return err
}
}
length := uint64(len(data))
n := binary.PutUvarint(this.lenBuf, length)
_, err = this.w.Write(this.lenBuf[:n])
if err != nil {
return err
}
_, err = this.w.Write(data)
return err
}
func TestGetExtensionStability(t *testing.T) {
check := func(m *pb.MyMessage) bool {
ext1, err := proto.GetExtension(m, pb.E_Ext_More)
if err != nil {
t.Fatalf("GetExtension() failed: %s", err)
}
ext2, err := proto.GetExtension(m, pb.E_Ext_More)
if err != nil {
t.Fatalf("GetExtension() failed: %s", err)
}
return ext1 == ext2
}
msg := &pb.MyMessage{Count: proto.Int32(4)}
ext0 := &pb.Ext{}
if err := proto.SetExtension(msg, pb.E_Ext_More, ext0); err != nil {
t.Fatalf("Could not set ext1: %s", ext0)
}
if !check(msg) {
t.Errorf("GetExtension() not stable before marshaling")
}
bb, err := proto.Marshal(msg)
if err != nil {
t.Fatalf("Marshal() failed: %s", err)
}
msg1 := &pb.MyMessage{}
err = proto.Unmarshal(bb, msg1)
if err != nil {
t.Fatalf("Unmarshal() failed: %s", err)
}
if !check(msg1) {
t.Errorf("GetExtension() not stable after unmarshaling")
}
}
func (n *FSNode) GetBytes() ([]byte, error) {
pbn := new(pb.Data)
pbn.Type = &n.Type
pbn.Filesize = proto.Uint64(uint64(len(n.Data)) + n.subtotal)
pbn.Blocksizes = n.blocksizes
pbn.Data = n.Data
return proto.Marshal(pbn)
}
// putLocal stores the key value pair in the datastore
func (dht *IpfsDHT) putLocal(key key.Key, rec *pb.Record) error {
data, err := proto.Marshal(rec)
if err != nil {
return err
}
return dht.datastore.Put(key.DsKey(), data)
}
func (sk *RsaPrivateKey) Bytes() ([]byte, error) {
b := x509.MarshalPKCS1PrivateKey(sk.sk)
pbmes := new(pb.PrivateKey)
typ := pb.KeyType_RSA
pbmes.Type = &typ
pbmes.Data = b
return proto.Marshal(pbmes)
}
// MarshalPrivateKey converts a key object into its protobuf serialized form.
func MarshalPrivateKey(k PrivKey) ([]byte, error) {
b := MarshalRsaPrivateKey(k.(*RsaPrivateKey))
pmes := new(pb.PrivateKey)
typ := pb.KeyType_RSA // for now only type.
pmes.Type = &typ
pmes.Data = b
return proto.Marshal(pmes)
}
func newTestMessage() *pb.MyMessage {
msg := &pb.MyMessage{
Count: proto.Int32(42),
Name: proto.String("Dave"),
Quote: proto.String(`"I didn't want to go."`),
Pet: []string{"bunny", "kitty", "horsey"},
Inner: &pb.InnerMessage{
Host: proto.String("footrest.syd"),
Port: proto.Int32(7001),
Connected: proto.Bool(true),
},
Others: []*pb.OtherMessage{
{
Key: proto.Int64(0xdeadbeef),
Value: []byte{1, 65, 7, 12},
},
{
Weight: proto.Float32(6.022),
Inner: &pb.InnerMessage{
Host: proto.String("lesha.mtv"),
Port: proto.Int32(8002),
},
},
},
Bikeshed: pb.MyMessage_BLUE.Enum(),
Somegroup: &pb.MyMessage_SomeGroup{
GroupField: proto.Int32(8),
},
// One normally wouldn't do this.
// This is an undeclared tag 13, as a varint (wire type 0) with value 4.
XXX_unrecognized: []byte{13<<3 | 0, 4},
}
ext := &pb.Ext{
Data: proto.String("Big gobs for big rats"),
}
if err := proto.SetExtension(msg, pb.E_Ext_More, ext); err != nil {
panic(err)
}
greetings := []string{"adg", "easy", "cow"}
if err := proto.SetExtension(msg, pb.E_Greeting, greetings); err != nil {
panic(err)
}
// Add an unknown extension. We marshal a pb.Ext, and fake the ID.
b, err := proto.Marshal(&pb.Ext{Data: proto.String("3G skiing")})
if err != nil {
panic(err)
}
b = append(proto.EncodeVarint(201<<3|proto.WireBytes), b...)
proto.SetRawExtension(msg, 201, b)
// Extensions can be plain fields, too, so let's test that.
b = append(proto.EncodeVarint(202<<3|proto.WireVarint), 19)
proto.SetRawExtension(msg, 202, b)
return msg
}
// MarshalPublicKey converts a public key object into a protobuf serialized
// public key
func MarshalPublicKey(k PubKey) ([]byte, error) {
b, err := MarshalRsaPublicKey(k.(*RsaPublicKey))
if err != nil {
return nil, err
}
pmes := new(pb.PublicKey)
typ := pb.KeyType_RSA // for now only type.
pmes.Type = &typ
pmes.Data = b
return proto.Marshal(pmes)
}
func (c *offlineRouting) PutValue(ctx context.Context, key key.Key, val []byte) error {
rec, err := record.MakePutRecord(c.sk, key, val, false)
if err != nil {
return err
}
data, err := proto.Marshal(rec)
if err != nil {
return err
}
return c.datastore.Put(key.DsKey(), data)
}
func (pk *RsaPublicKey) Bytes() ([]byte, error) {
b, err := x509.MarshalPKIXPublicKey(pk.k)
if err != nil {
return nil, err
}
pbmes := new(pb.PublicKey)
typ := pb.KeyType_RSA
pbmes.Type = &typ
pbmes.Data = b
return proto.Marshal(pbmes)
}
// Returns Bytes that represent a Directory
func FolderPBData() []byte {
pbfile := new(pb.Data)
typ := pb.Data_Directory
pbfile.Type = &typ
data, err := proto.Marshal(pbfile)
if err != nil {
//this really shouldnt happen, i promise
panic(err)
}
return data
}
func putRoutingRecord(ds datastore.Datastore, k key.Key, value *dhtpb.Record) error {
data, err := proto.Marshal(value)
if err != nil {
return err
}
dskey := k.DsKey()
// TODO namespace
if err := ds.Put(dskey, data); err != nil {
return err
}
return nil
}
func SymlinkData(path string) ([]byte, error) {
pbdata := new(pb.Data)
typ := pb.Data_Symlink
pbdata.Data = []byte(path)
pbdata.Type = &typ
out, err := proto.Marshal(pbdata)
if err != nil {
return nil, err
}
return out, nil
}
func BytesForMetadata(m *Metadata) ([]byte, error) {
pbd := new(pb.Data)
pbd.Filesize = proto.Uint64(m.Size)
typ := pb.Data_Metadata
pbd.Type = &typ
mdd, err := m.Bytes()
if err != nil {
return nil, err
}
pbd.Data = mdd
return proto.Marshal(pbd)
}
func WrapData(b []byte) []byte {
pbdata := new(pb.Data)
typ := pb.Data_Raw
pbdata.Data = b
pbdata.Type = &typ
out, err := proto.Marshal(pbdata)
if err != nil {
// This shouldnt happen. seriously.
panic(err)
}
return out
}
func createRoutingEntryData(pk ci.PrivKey, val path.Path) ([]byte, error) {
entry := new(pb.IpnsEntry)
entry.Value = []byte(val)
typ := pb.IpnsEntry_EOL
entry.ValidityType = &typ
entry.Validity = []byte(u.FormatRFC3339(time.Now().Add(time.Hour * 24)))
sig, err := pk.Sign(ipnsEntryDataForSig(entry))
if err != nil {
return nil, err
}
entry.Signature = sig
return proto.Marshal(entry)
}
func TestPBNodeVerboseEqual(t *testing.T) {
popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
p := NewPopulatedPBNode(popr, false)
data, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
panic(err)
}
msg := &PBNode{}
if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil {
panic(err)
}
if err := p.VerboseEqual(msg); err != nil {
t.Fatalf("%#v !VerboseEqual %#v, since %v", msg, p, err)
}
}
func BenchmarkPBLinkProtoMarshal(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
pops := make([]*PBLink, 10000)
for i := 0; i < 10000; i++ {
pops[i] = NewPopulatedPBLink(popr, false)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
data, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000])
if err != nil {
panic(err)
}
total += len(data)
}
b.SetBytes(int64(total / b.N))
}
// Store a value in this peer local storage
func (dht *IpfsDHT) handlePutValue(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
defer log.EventBegin(ctx, "handlePutValue", p).Done()
dskey := key.Key(pmes.GetKey()).DsKey()
if err := dht.verifyRecordLocally(pmes.GetRecord()); err != nil {
log.Debugf("Bad dht record in PUT from: %s. %s", key.Key(pmes.GetRecord().GetAuthor()), err)
return nil, err
}
data, err := proto.Marshal(pmes.GetRecord())
if err != nil {
return nil, err
}
err = dht.datastore.Put(dskey, data)
log.Debugf("%s handlePutValue %v", dht.self, dskey)
return pmes, err
}
func FilePBData(data []byte, totalsize uint64) []byte {
pbfile := new(pb.Data)
typ := pb.Data_File
pbfile.Type = &typ
pbfile.Data = data
pbfile.Filesize = proto.Uint64(totalsize)
data, err := proto.Marshal(pbfile)
if err != nil {
// This really shouldnt happen, i promise
// The only failure case for marshal is if required fields
// are not filled out, and they all are. If the proto object
// gets changed and nobody updates this function, the code
// should panic due to programmer error
panic(err)
}
return data
}
func TestPBNodeSize(t *testing.T) {
popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
p := NewPopulatedPBNode(popr, true)
size2 := github_com_gogo_protobuf_proto.Size(p)
data, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
panic(err)
}
size := p.Size()
if len(data) != size {
t.Fatalf("size %v != marshalled size %v", size, len(data))
}
if size2 != size {
t.Fatalf("size %v != before marshal proto.Size %v", size, size2)
}
size3 := github_com_gogo_protobuf_proto.Size(p)
if size3 != size {
t.Fatalf("size %v != after marshal proto.Size %v", size, size3)
}
}
func TestPBLinkProto(t *testing.T) {
popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
p := NewPopulatedPBLink(popr, false)
data, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
panic(err)
}
msg := &PBLink{}
if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil {
panic(err)
}
for i := range data {
data[i] = byte(popr.Intn(256))
}
if err := p.VerboseEqual(msg); err != nil {
t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err)
}
if !p.Equal(msg) {
t.Fatalf("%#v !Proto %#v", msg, p)
}
}
func (this *fullWriter) WriteMsg(msg proto.Message) (err error) {
var data []byte
if m, ok := msg.(marshaler); ok {
n := m.Size()
if n >= len(this.buffer) {
this.buffer = make([]byte, n)
}
_, err = m.MarshalTo(this.buffer)
if err != nil {
return err
}
data = this.buffer[:n]
} else {
data, err = proto.Marshal(msg)
if err != nil {
return err
}
}
_, err = this.w.Write(data)
return err
}
func BenchmarkPBLinkProtoUnmarshal(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
datas := make([][]byte, 10000)
for i := 0; i < 10000; i++ {
data, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedPBLink(popr, false))
if err != nil {
panic(err)
}
datas[i] = data
}
msg := &PBLink{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
total += len(datas[i%10000])
if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil {
panic(err)
}
}
b.SetBytes(int64(total / b.N))
}
// writeMsgCtx is used by the
func writeMsgCtx(ctx context.Context, w msgio.Writer, msg proto.Message) ([]byte, error) {
enc, err := proto.Marshal(msg)
if err != nil {
return nil, err
}
// write in a goroutine so we can exit when our context is cancelled.
done := make(chan error)
go func(m []byte) {
err := w.WriteMsg(m)
select {
case done <- err:
case <-ctx.Done():
}
}(enc)
select {
case <-ctx.Done():
return nil, ctx.Err()
case e := <-done:
return enc, e
}
}
func putRoutingProviders(ds datastore.Datastore, k key.Key, newRecords []*dhtpb.Message_Peer) error {
log.Event(context.Background(), "putRoutingProviders", &k)
oldRecords, err := getRoutingProviders(ds, k)
if err != nil {
return err
}
mergedRecords := make(map[string]*dhtpb.Message_Peer)
for _, provider := range oldRecords {
mergedRecords[provider.GetId()] = provider // add original records
}
for _, provider := range newRecords {
mergedRecords[provider.GetId()] = provider // overwrite old record if new exists
}
var protomsg dhtpb.Message
protomsg.ProviderPeers = make([]*dhtpb.Message_Peer, 0, len(mergedRecords))
for _, provider := range mergedRecords {
protomsg.ProviderPeers = append(protomsg.ProviderPeers, provider)
}
data, err := proto.Marshal(&protomsg)
if err != nil {
return err
}
return ds.Put(providerKey(k), data)
}
// modifyDag writes the data in 'data' over the data in 'node' starting at 'offset'
// returns the new key of the passed in node and whether or not all the data in the reader
// has been consumed.
func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader) (key.Key, bool, error) {
f, err := ft.FromBytes(node.Data)
if err != nil {
return "", false, err
}
// If we've reached a leaf node.
if len(node.Links) == 0 {
n, err := data.Read(f.Data[offset:])
if err != nil && err != io.EOF {
return "", false, err
}
// Update newly written node..
b, err := proto.Marshal(f)
if err != nil {
return "", false, err
}
nd := &mdag.Node{Data: b}
k, err := dm.dagserv.Add(nd)
if err != nil {
return "", false, err
}
// Hey look! we're done!
var done bool
if n < len(f.Data[offset:]) {
done = true
}
return k, done, nil
}
var cur uint64
var done bool
for i, bs := range f.GetBlocksizes() {
// We found the correct child to write into
if cur+bs > offset {
// Unpin block
ckey := key.Key(node.Links[i].Hash)
dm.mp.RemovePinWithMode(ckey, pin.Indirect)
child, err := node.Links[i].GetNode(dm.ctx, dm.dagserv)
if err != nil {
return "", false, err
}
k, sdone, err := dm.modifyDag(child, offset-cur, data)
if err != nil {
return "", false, err
}
// pin the new node
dm.mp.PinWithMode(k, pin.Indirect)
offset += bs
node.Links[i].Hash = mh.Multihash(k)
// Recache serialized node
_, err = node.Encoded(true)
if err != nil {
return "", false, err
}
if sdone {
// No more bytes to write!
done = true
break
}
offset = cur + bs
}
cur += bs
}
k, err := dm.dagserv.Add(node)
return k, done, err
}
func (m *Metadata) Bytes() ([]byte, error) {
pbm := new(pb.Metadata)
pbm.MimeType = &m.MimeType
return proto.Marshal(pbm)
}