// PutMulti is a batch version of Put. One PendingKey is returned for each
// element of src in the same order.
func (t *Transaction) PutMulti(keys []*Key, src interface{}) ([]*PendingKey, error) {
if t.id == nil {
return nil, errExpiredTransaction
}
mutation, err := putMutation(keys, src)
if err != nil {
return nil, err
}
proto.Merge(t.mutation, mutation)
// Prepare the returned handles, pre-populating where possible.
ret := make([]*PendingKey, len(keys))
for i, key := range keys {
h := &PendingKey{}
if key.Incomplete() {
// This key will be in the final commit result.
t.pending = append(t.pending, h)
} else {
h.key = key
}
ret[i] = h
}
return ret, nil
}
// Get a list of all members which are currently in the trash.
func (m *MembershipDB) EnumerateTrashedMembers(prev string, num int32) ([]*MemberWithKey, error) {
var cp *cassandra.ColumnParent = cassandra.NewColumnParent()
var pred *cassandra.SlicePredicate = cassandra.NewSlicePredicate()
var r *cassandra.KeyRange = cassandra.NewKeyRange()
var kss []*cassandra.KeySlice
var ks *cassandra.KeySlice
var rv []*MemberWithKey
var err error
// Fetch the protobuf column of the application column family.
cp.ColumnFamily = "membership_archive"
pred.ColumnNames = [][]byte{
[]byte("pb_data"),
}
if len(prev) > 0 {
var uuid cassandra.UUID
if uuid, err = cassandra.ParseUUID(prev); err != nil {
return rv, err
}
r.StartKey = append([]byte(archivePrefix), []byte(uuid)...)
} else {
r.StartKey = []byte(archivePrefix)
}
r.EndKey = []byte(archiveEnd)
r.Count = num
kss, err = m.conn.GetRangeSlices(cp, pred, r, cassandra.ConsistencyLevel_ONE)
if err != nil {
return rv, err
}
for _, ks = range kss {
var member *MemberWithKey
var scol *cassandra.ColumnOrSuperColumn
var uuid cassandra.UUID = cassandra.UUIDFromBytes(
ks.Key[len(archivePrefix):])
if len(ks.Columns) == 0 {
continue
}
for _, scol = range ks.Columns {
var col *cassandra.Column = scol.Column
if string(col.Name) == "pb_data" {
var agreement = new(MembershipAgreement)
member = new(MemberWithKey)
err = proto.Unmarshal(col.Value, agreement)
proto.Merge(&member.Member, agreement.GetMemberData())
member.Key = uuid.String()
}
}
if member != nil {
rv = append(rv, member)
}
}
return rv, nil
}
func (t testProtoTable) Lookup(_ context.Context, key []byte, msg proto.Message) error {
m, ok := t[string(key)]
if !ok {
return table.ErrNoSuchKey
}
proto.Merge(msg, m)
return nil
}
func (t testProtoTable) Lookup(_ context.Context, key []byte, msg proto.Message) error {
m, ok := t[string(key)]
if !ok {
log.Println("Failed to find key", string(key))
return table.ErrNoSuchKey
}
proto.Merge(msg, m)
return nil
}
func TestMerge(t *testing.T) {
for _, m := range mergeTests {
got := proto.Clone(m.dst)
proto.Merge(got, m.src)
if !proto.Equal(got, m.want) {
t.Errorf("Merge(%v, %v)\n got %v\nwant %v\n", m.dst, m.src, got, m.want)
}
}
}
// DeleteMulti is a batch version of Delete.
func (t *Transaction) DeleteMulti(keys []*Key) error {
if t.id == nil {
return errExpiredTransaction
}
mutation, err := deleteMutation(keys)
if err != nil {
return err
}
proto.Merge(t.mutation, mutation)
return nil
}
func TestProtoMerge(t *testing.T) {
person := InitData("ronaflx", 195936, "*****@*****.**")
merged := InitData("flx", 1990, "*****@*****.**")
proto.Merge(merged, person) // The first param is destination
if merged.GetName() != "ronaflx" {
t.Error("expect merged.name == ronaflx", merged.GetName())
}
if merged.GetId() != 195936 {
t.Error("expect merged.id == 195936", merged.GetId())
}
if merged.GetEmail() != "*****@*****.**" {
t.Error("expect merged.email == [email protected]", merged.GetEmail())
}
}
func (m *MaxReplicationLagModule) updateConfiguration(configuration *throttlerdata.Configuration, copyZeroValues bool) error {
m.mutableConfigMu.Lock()
defer m.mutableConfigMu.Unlock()
newConfig := m.mutableConfig
if copyZeroValues {
newConfig.Configuration = *proto.Clone(configuration).(*throttlerdata.Configuration)
} else {
proto.Merge(&newConfig.Configuration, configuration)
}
if err := newConfig.Verify(); err != nil {
return err
}
m.mutableConfig = newConfig
m.applyMutableConfig = true
return nil
}
func (fs *AppendFS) LoadMetadata() error {
ret := (error)(nil)
fs.metadataMutex.Lock()
nodes := make(map[uint64]*messages.NodeMetadata)
children := make(map[uint64][]uint64)
_, err := fs.metadataFile.Seek(0, 0)
bufferedReader := bufio.NewReader(fs.metadataFile)
if err != nil {
ret = err
goto Finally
}
for {
msgLen, err := binary.ReadUvarint(bufferedReader)
if err != nil {
// EOF is ok here
if err == io.EOF {
fmt.Println("Reached expected EOF")
break
}
ret = err
goto Finally
}
msgBuf := make([]byte, msgLen)
//fmt.Printf("MsgLen: %d\n", msgLen)
_, err = io.ReadAtLeast(bufferedReader, msgBuf, int(msgLen))
if err != nil {
ret = err
goto Finally
}
metadata := &messages.NodeMetadata{}
err = proto.Unmarshal(msgBuf, metadata)
if err != nil {
ret = err
goto Finally
}
if currentNode, ok := nodes[metadata.GetNodeId()]; ok {
if metadata.Contents != nil {
currentNode.Contents = nil
}
proto.Merge(currentNode, metadata)
//fmt.Printf("Merged %d\n", currentNode.GetNodeId())
} else {
nodes[metadata.GetNodeId()] = metadata
//fmt.Printf("Added %d\n", metadata.GetNodeId())
}
}
for id, node := range nodes {
fs.seenNodeId(id)
if !node.GetValid() {
continue
}
if node.ParentNodeId != nil {
if currentChildren, ok := children[node.GetParentNodeId()]; ok {
children[node.GetParentNodeId()] = append(currentChildren, id)
} else {
children[node.GetParentNodeId()] = append(make([]uint64, 0), id)
}
} else {
ret = errors.New("Corrupt metadata: missing ParentFileId for file " + string(id))
goto Finally
}
}
fs.addChildrenHelper(nodes, children, fs.root)
Finally:
fs.metadataMutex.Unlock()
return ret
}
func TestUnmarshalRepeatingNonRepeatedExtension(t *testing.T) {
// We may see multiple instances of the same extension in the wire
// format. For example, the proto compiler may encode custom options in
// this way. Here, we verify that we merge the extensions together.
tests := []struct {
name string
ext []*pb.ComplexExtension
}{
{
"two fields",
[]*pb.ComplexExtension{
{First: proto.Int32(7)},
{Second: proto.Int32(11)},
},
},
{
"repeated field",
[]*pb.ComplexExtension{
{Third: []int32{1000}},
{Third: []int32{2000}},
},
},
{
"two fields and repeated field",
[]*pb.ComplexExtension{
{Third: []int32{1000}},
{First: proto.Int32(9)},
{Second: proto.Int32(21)},
{Third: []int32{2000}},
},
},
}
for _, test := range tests {
var buf bytes.Buffer
var want pb.ComplexExtension
// Generate a serialized representation of a repeated extension
// by catenating bytes together.
for i, e := range test.ext {
// Merge to create the wanted proto.
proto.Merge(&want, e)
// serialize the message
msg := new(pb.OtherMessage)
err := proto.SetExtension(msg, pb.E_Complex, e)
if err != nil {
t.Fatalf("[%s] Error setting extension %d: %v", test.name, i, err)
}
b, err := proto.Marshal(msg)
if err != nil {
t.Fatalf("[%s] Error marshaling message %d: %v", test.name, i, err)
}
buf.Write(b)
}
// Unmarshal and read the merged proto.
msg2 := new(pb.OtherMessage)
err := proto.Unmarshal(buf.Bytes(), msg2)
if err != nil {
t.Fatalf("[%s] Error unmarshaling message: %v", test.name, err)
}
e, err := proto.GetExtension(msg2, pb.E_Complex)
if err != nil {
t.Fatalf("[%s] Error getting extension: %v", test.name, err)
}
ext := e.(*pb.ComplexExtension)
if ext == nil {
t.Fatalf("[%s] Invalid extension", test.name)
}
if !reflect.DeepEqual(*ext, want) {
t.Errorf("[%s] Wrong value for ComplexExtension: got: %s want: %s\n", test.name, ext, want)
}
}
}
// Output a JSON list of all applicants currently waiting to become members.
func (a *ApplicantListHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
var applist applicantListType
var enc *json.Encoder
var err error
if !a.auth.IsAuthenticatedScope(req, a.admingroup) {
rw.WriteHeader(http.StatusUnauthorized)
return
}
if req.FormValue("single") == "true" && len(req.FormValue("start")) > 0 {
var memberreq *membersys.MembershipAgreement
var mwk *membersys.MemberWithKey
var bigint *big.Int = big.NewInt(0)
var uuid cassandra.UUID
var ok bool
bigint, ok = bigint.SetString(req.FormValue("start"), 10)
if !ok {
rw.WriteHeader(http.StatusInternalServerError)
rw.Write([]byte("Unable to parse " + req.FormValue("start") +
" as a number"))
return
}
if bigint.BitLen() == 128 {
uuid = cassandra.UUIDFromBytes(bigint.Bytes())
memberreq, _, err = a.database.GetMembershipRequest(
uuid.String(), "application", "applicant:")
if err != nil {
rw.WriteHeader(http.StatusInternalServerError)
rw.Write([]byte("Unable to retrieve the membership request " +
uuid.String() + ": " + err.Error()))
return
}
mwk = new(membersys.MemberWithKey)
mwk.Key = uuid.String()
proto.Merge(&mwk.Member, memberreq.GetMemberData())
applist.Applicants = []*membersys.MemberWithKey{mwk}
}
} else {
applist.Applicants, err = a.database.EnumerateMembershipRequests(
req.FormValue("criterion"), req.FormValue("start"), a.pagesize)
if err != nil {
log.Print("Error enumerating applications: ", err)
rw.WriteHeader(http.StatusInternalServerError)
rw.Write([]byte("Error enumerating applications: " + err.Error()))
return
}
}
applist.AgreementUploadCsrfToken, err = a.auth.GenCSRFToken(
req, applicantAgreementUploadURL, 10*time.Minute)
if err != nil {
log.Print("Error generating CSRF token: ", err)
rw.WriteHeader(http.StatusInternalServerError)
rw.Write([]byte("Error generating CSRF token: " + err.Error()))
return
}
applist.ApprovalCsrfToken, err = a.auth.GenCSRFToken(
req, applicantApprovalURL, 10*time.Minute)
if err != nil {
log.Print("Error generating CSRF token: ", err)
rw.WriteHeader(http.StatusInternalServerError)
rw.Write([]byte("Error generating CSRF token: " + err.Error()))
return
}
applist.RejectionCsrfToken, err = a.auth.GenCSRFToken(
req, applicantRejectionURL, 10*time.Minute)
if err != nil {
log.Print("Error generating CSRF token: ", err)
rw.WriteHeader(http.StatusInternalServerError)
rw.Write([]byte("Error generating CSRF token: " + err.Error()))
return
}
rw.Header().Set("Content-Type", "application/json; encoding=utf8")
enc = json.NewEncoder(rw)
if err = enc.Encode(applist); err != nil {
log.Print("Error JSON encoding member list: ", err)
rw.WriteHeader(http.StatusInternalServerError)
rw.Write([]byte("Error encoding result: " + err.Error()))
return
}
}
func fuzz(data []byte, text bool) int {
ctors := []func() proto.Message{
func() proto.Message { return new(pb.M0) },
func() proto.Message { return new(pb.M1) },
func() proto.Message { return new(pb.M2) },
func() proto.Message { return new(pb.M3) },
func() proto.Message { return new(pb.M4) },
func() proto.Message { return new(pb.M5) },
func() proto.Message { return new(pb.M6) },
func() proto.Message { return new(pb.M7) },
func() proto.Message { return new(pb.M8) },
func() proto.Message { return new(pb.M9) },
func() proto.Message { return new(pb.M10) },
func() proto.Message { return new(pb.M11) },
func() proto.Message { return new(pb.M12) },
func() proto.Message { return new(pb.M13) },
func() proto.Message { return new(pb.M14) },
func() proto.Message { return new(pb.M15) },
func() proto.Message { return new(pb.M16) },
func() proto.Message { return new(pb.M17) },
func() proto.Message { return new(pb.M18) },
func() proto.Message { return new(pb.M19) },
func() proto.Message { return new(pb.M20) },
func() proto.Message { return new(pb.M21) },
func() proto.Message { return new(pb.M22) },
func() proto.Message { return new(pb.M23) },
func() proto.Message { return new(pb.M24) },
func() proto.Message { return new(pb.M25) },
}
datas := ""
if text {
datas = string(data)
}
score := 0
for _, ctor := range ctors {
v := ctor()
var err error
if text {
err = proto.UnmarshalText(datas, v)
} else {
err = proto.Unmarshal(data, v)
}
if err != nil {
continue
}
score = 1
sz := proto.Size(v)
var data1 []byte
if text {
var buf bytes.Buffer
err = proto.MarshalText(&buf, v)
data1 = buf.Bytes()
} else {
data1, err = proto.Marshal(v)
}
if err != nil {
panic(err)
}
v1 := ctor()
if text {
err = proto.UnmarshalText(string(data1), v1)
} else {
if sz != len(data1) {
panic(fmt.Sprintf("Size returned %v, while Marshal returned %v", sz, len(data1)))
}
err = proto.Unmarshal(data1, v1)
}
if err != nil {
panic(err)
}
if !DeepEqual(v, v1) {
fmt.Printf("v0: %#v\n", v)
fmt.Printf("v1: %#v\n", v1)
panic(fmt.Sprintf("non idempotent marshal of %T", v))
}
if !proto.Equal(v, v1) {
fmt.Printf("v0: %#v\n", v)
fmt.Printf("v1: %#v\n", v1)
panic(fmt.Sprintf("non idempotent marshal of %T", v))
}
if text {
// TODO: Marshal/Unmarshal to binary.
} else {
var buf bytes.Buffer
err := proto.MarshalText(&buf, v)
if err != nil {
fmt.Printf("failed to MarshalText: %#v\n", v)
panic(err)
}
v2 := ctor()
err = proto.UnmarshalText(buf.String(), v2)
if err != nil {
if strings.Contains(err.Error(), "unexpected byte 0x2f") {
continue // known bug
}
fmt.Printf("failed to UnmarshalText: %q\n", buf.Bytes())
panic(err)
}
if !proto.Equal(v, v2) {
fmt.Printf("v0: %#v\n", v)
fmt.Printf("v2: %#v\n", v2)
panic(fmt.Sprintf("non idempotent text marshal of %T", v))
}
}
v3 := proto.Clone(v)
if !DeepEqual(v, v3) {
fmt.Printf("v0: %#v\n", v)
fmt.Printf("v3: %#v\n", v3)
panic(fmt.Sprintf("bad clone of %T", v))
}
proto.SetDefaults(v3)
if !proto.Equal(v, v3) {
fmt.Printf("v0: %#v\n", v)
fmt.Printf("v3: %#v\n", v3)
panic(fmt.Sprintf("SetDefaults changed data %T", v))
}
proto.Merge(v3, v1)
if idempotentMerge(v) && !proto.Equal(v, v3) {
fmt.Printf("v0: %#v\n", v)
fmt.Printf("v3: %#v\n", v3)
panic(fmt.Sprintf("Merge changed data %T", v))
}
}
return score
}