func Test002ListListStructList(t *testing.T) { cv.Convey("Given type RWTest struct { NestMatrix [][]Nester1; } in go, where Nester1 is a struct, and a mirror/parallel capnp struct air.RWTestCapn { nestMatrix @0: List(List(Nester1Capn)); } defined in the aircraftlib schema", t, func() { cv.Convey("When we Save() RWTest to capn and then Load() it back, the data should match, so that we have working List(List(Struct)) serialization and deserializatoin in go-capnproto", func() { // full RWTest rw := RWTest{ NestMatrix: [][]Nester1{[]Nester1{Nester1{Strs: []string{"z", "w"}}, Nester1{Strs: []string{"q", "r"}}}, []Nester1{Nester1{Strs: []string{"zebra", "wally"}}, Nester1{Strs: []string{"qubert", "rocks"}}}}, } var o bytes.Buffer rw.Save(&o) seg, n, err := capn.ReadFromMemoryZeroCopy(o.Bytes()) cv.So(err, cv.ShouldEqual, nil) cv.So(n, cv.ShouldBeGreaterThan, 0) text := CapnpDecodeSegment(seg, "", "aircraftlib/aircraft.capnp", "RWTestCapn") if false { fmt.Printf("text = '%s'\n", text) } rw2 := &RWTest{} rw2.Load(&o) //fmt.Printf("rw = '%#v'\n", rw) //fmt.Printf("rw2 = '%#v'\n", rw2) same := reflect.DeepEqual(&rw, rw2) cv.So(same, cv.ShouldEqual, true) }) }) }
// start with smaller Struct(List) func Test001StructList(t *testing.T) { cv.Convey("Given type Nester1 struct { Strs []string } in go, where Nester1 is a struct, and a mirror/parallel capnp struct air.Nester1Capn { strs @0: List(Text); } defined in the aircraftlib schema", t, func() { cv.Convey("When we Save() Nester to capn and then Load() it back, the data should match, so that we have working Struct(List) serialization and deserializatoin in go-capnproto", func() { // Does Nester1 alone serialization and deser okay? rw := Nester1{Strs: []string{"xenophilia", "watchowski"}} var o bytes.Buffer rw.Save(&o) seg, n, err := capn.ReadFromMemoryZeroCopy(o.Bytes()) cv.So(err, cv.ShouldEqual, nil) cv.So(n, cv.ShouldBeGreaterThan, 0) text := CapnpDecodeSegment(seg, "", "aircraftlib/aircraft.capnp", "Nester1Capn") if false { fmt.Printf("text = '%s'\n", text) } rw2 := &Nester1{} rw2.Load(&o) //fmt.Printf("rw = '%#v'\n", rw) //fmt.Printf("rw2 = '%#v'\n", rw2) same := reflect.DeepEqual(&rw, rw2) cv.So(same, cv.ShouldEqual, true) }) }) }
// disk file of a capn segment -> in-memory capn segment -> stdin to capnp decode -> stdout human-readble string form func CapnFileToText(serializedCapnpFilePathToDisplay string, capnpSchemaFilePath string, capnpExePath string) (string, error) { // a) read file into Segment byteslice, err := ioutil.ReadFile(serializedCapnpFilePathToDisplay) if err != nil { return "", err } seg, nbytes, err := capn.ReadFromMemoryZeroCopy(byteslice) if err == io.EOF { return "", err } if err != nil { return "", err } if nbytes == 0 { return "", errors.New(fmt.Sprintf("did not expect 0 bytes back from capn.ReadFromMemoryZeroCopy() on reading file '%s'", serializedCapnpFilePathToDisplay)) } // b) tell CapnpDecodeSegment() to show the human-readable-text form of the message // warning: CapnpDecodeSegment() may panic on you. It is a testing utility so that // is desirable. For production, do something else. return CapnpDecodeSegment(seg, capnpExePath, capnpSchemaFilePath, "Z"), nil }
func ProposerFromData(pm *ProposerManager, txnId *common.TxnId, data []byte) *Proposer { seg, _, err := capn.ReadFromMemoryZeroCopy(data) if err != nil { log.Println("Unable to decode proposer state", data) } // If we were on disk, then that means we must be locally complete // and just need to send out TLCs. state := msgs.ReadRootProposerState(seg) acceptorsCap := state.Acceptors() acceptors := make([]common.RMId, acceptorsCap.Len()) for idx := range acceptors { acceptors[idx] = common.RMId(acceptorsCap.At(idx)) } // We were on disk. Thus we received outcomes from all // acceptors. So we don't need to worry about the outcome // accumulator's fInc, hence just use -1 here. p := &Proposer{ proposerManager: pm, mode: proposerTLCSender, txnId: txnId, acceptors: acceptors, fInc: -1, } p.init() p.allAcceptorsAgreed = true return p }
func VarFromData(data []byte, exe *dispatcher.Executor, disk *mdbs.MDBServer, vm *VarManager) (*Var, error) { seg, _, err := capn.ReadFromMemoryZeroCopy(data) if err != nil { return nil, err } varCap := msgs.ReadRootVar(seg) v := newVar(common.MakeVarUUId(varCap.Id()), exe, disk, vm) positions := varCap.Positions() if positions.Len() != 0 { v.positions = (*common.Positions)(&positions) } writeTxnId := common.MakeTxnId(varCap.WriteTxnId()) writeTxnClock := VectorClockFromCap(varCap.WriteTxnClock()) writesClock := VectorClockFromCap(varCap.WritesClock()) server.Log(v.UUId, "Restored", writeTxnId) if result, err := disk.ReadonlyTransaction(func(rtxn *mdbs.RTxn) (interface{}, error) { return db.ReadTxnFromDisk(rtxn, writeTxnId) }).ResultError(); err == nil { if result == nil || result.(*msgs.Txn) == nil { panic(fmt.Sprintf("%v Unable to find txn %v on disk (%v)", v.UUId, writeTxnId, result)) } actions := result.(*msgs.Txn).Actions() v.curFrame = NewFrame(nil, v, writeTxnId, &actions, writeTxnClock, writesClock) v.curFrameOnDisk = v.curFrame } else { return nil, err } v.varCap = &varCap return v, nil }
func (cah *connectionAwaitHandshake) readAndDecryptOne() (*capn.Segment, error) { if cah.sessionKey == nil { return capn.ReadFromStream(cah.socket, nil) } read, err := cah.socket.Read(cah.inBuff) if err != nil { return nil, err } else if read < len(cah.inBuff) { return nil, fmt.Errorf("Only read %v bytes, wanted %v", read, len(cah.inBuff)) } copy(cah.nonceAryIn[16:], cah.inBuff[:8]) msgLen := binary.BigEndian.Uint64(cah.inBuff[8:16]) plainLen := msgLen - secretbox.Overhead msgBuf := make([]byte, plainLen+msgLen) for recvBuf := msgBuf[plainLen:]; len(recvBuf) != 0; { read, err = cah.socket.Read(recvBuf) if err != nil { return nil, err } else { recvBuf = recvBuf[read:] } } plaintext, ok := secretbox.Open(msgBuf[:0], msgBuf[plainLen:], cah.nonceAryIn, cah.sessionKey) if !ok { return nil, fmt.Errorf("Unable to decrypt message") } seg, _, err := capn.ReadFromMemoryZeroCopy(plaintext) return seg, err }
func TopologyDeserialize(txnId *common.TxnId, root *msgs.VarIdPos, data []byte) (*Topology, error) { seg, _, err := capn.ReadFromMemoryZeroCopy(data) if err != nil { return nil, err } topology := msgs.ReadRootTopology(seg) return TopologyFromCap(txnId, root, &topology), nil }
func TopologyFromCap(txnId *common.TxnId, root *msgs.VarIdPos, data []byte) (*Topology, error) { seg, _, err := capn.ReadFromMemoryZeroCopy(data) if err != nil { return nil, err } configCap := msgs.ReadRootConfiguration(seg) config := ConfigurationFromCap(&configCap) return NewTopology(txnId, root, config), nil }
func TestDataVersioningZeroPointersToMore(t *testing.T) { expEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerEmptyList") cv.Convey("Given a struct with 0 ptr fields, and a newer version of the struct with 1-2 pointer fields", t, func() { cv.Convey("then serializing the empty list and reading it back into 1 or 2 pointer fields should default initialize the pointer fields", func() { seg := capn.NewBuffer(nil) scratch := capn.NewBuffer(nil) emptyholder := air.NewRootHoldsVerEmptyList(seg) elist := air.NewVerEmptyList(scratch, 2) emptyholder.SetMylist(elist) actEmpty := ShowSeg(" after NewRootHoldsVerEmptyList(seg) and SetMylist(elist), segment seg is:", seg) actEmptyCap := string(CapnpDecode(actEmpty, "HoldsVerEmptyList")) expEmptyCap := string(CapnpDecode(expEmpty, "HoldsVerEmptyList")) cv.So(actEmptyCap, cv.ShouldResemble, expEmptyCap) fmt.Printf("\n actEmpty is \n") ShowBytes(actEmpty, 10) fmt.Printf("actEmpty decoded by capnp: '%s'\n", string(CapnpDecode(actEmpty, "HoldsVerEmptyList"))) cv.So(actEmpty, cv.ShouldResemble, expEmpty) // seg is set, now read into bigger list buf := bytes.Buffer{} seg.WriteTo(&buf) segbytes := buf.Bytes() reseg, _, err := capn.ReadFromMemoryZeroCopy(segbytes) if err != nil { panic(err) } ShowSeg(" after re-reading segbytes, segment reseg is:", reseg) fmt.Printf("segbytes decoded by capnp as HoldsVerOneDataList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerOneDataList"))) reHolder := air.ReadRootHoldsVerTwoTwoList(reseg) list22 := reHolder.Mylist() len22 := list22.Len() cv.So(len22, cv.ShouldEqual, 2) for i := 0; i < 2; i++ { ele := list22.At(i) val := ele.Val() cv.So(val, cv.ShouldEqual, 0) duo := ele.Duo() cv.So(duo, cv.ShouldEqual, 0) ptr1 := ele.Ptr1() ptr2 := ele.Ptr2() fmt.Printf("ptr1 = %#v\n", ptr1) cv.So(ptr1.Segment, cv.ShouldEqual, nil) fmt.Printf("ptr2 = %#v\n", ptr2) cv.So(ptr2.Segment, cv.ShouldEqual, nil) } }) }) }
func VarFromData(data []byte, exe *dispatcher.Executor, db *db.Databases, vm *VarManager) (*Var, error) { seg, _, err := capn.ReadFromMemoryZeroCopy(data) if err != nil { return nil, err } varCap := msgs.ReadRootVar(seg) v := newVar(common.MakeVarUUId(varCap.Id()), exe, db, vm) positions := varCap.Positions() if positions.Len() != 0 { v.positions = (*common.Positions)(&positions) } writeTxnId := common.MakeTxnId(varCap.WriteTxnId()) writeTxnClock := VectorClockFromCap(varCap.WriteTxnClock()) writesClock := VectorClockFromCap(varCap.WritesClock()) server.Log(v.UUId, "Restored", writeTxnId) if result, err := db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} { return db.ReadTxnBytesFromDisk(rtxn, writeTxnId) }).ResultError(); err == nil && result != nil { bites := result.([]byte) if seg, _, err := capn.ReadFromMemoryZeroCopy(bites); err == nil { txn := msgs.ReadRootTxn(seg) actions := txn.Actions() v.curFrame = NewFrame(nil, v, writeTxnId, &actions, writeTxnClock, writesClock) v.curFrameOnDisk = v.curFrame v.varCap = &varCap return v, nil } else { return nil, err } } else { return nil, err } }
func loadVars(disk *mdbs.MDBServer, vars map[common.VarUUId]*varstate) { _, err := disk.ReadonlyTransaction(func(rtxn *mdbs.RTxn) (interface{}, error) { return rtxn.WithCursor(db.DB.Vars, func(cursor *mdb.Cursor) (interface{}, error) { key, data, err := cursor.Get(nil, nil, mdb.FIRST) for ; err == nil; key, data, err = cursor.Get(nil, nil, mdb.NEXT) { vUUId := common.MakeVarUUId(key) seg, _, err := capn.ReadFromMemoryZeroCopy(data) if err != nil { log.Println(err) continue } varCap := msgs.ReadRootVar(seg) pos := varCap.Positions() positions := (*common.Positions)(&pos) writeTxnId := common.MakeTxnId(varCap.WriteTxnId()) writeTxnClock := eng.VectorClockFromCap(varCap.WriteTxnClock()) writesClock := eng.VectorClockFromCap(varCap.WritesClock()) if state, found := vars[*vUUId]; found { if err := state.matches(disk, writeTxnId, writeTxnClock, writesClock, positions); err != nil { log.Println(err) } } else { state = &varstate{ vUUId: vUUId, disks: []*mdbs.MDBServer{disk}, writeTxnId: writeTxnId, writeTxnClock: writeTxnClock, writeWritesClock: writesClock, positions: positions, } vars[*vUUId] = state } } if err == mdb.NotFound { return nil, nil } else { return nil, err } }) }).ResultError() if err != nil { log.Println(err) } }
func (vw *varWrapper) start() { defer close(vw.c) c1 := &varWrapperCell{varWrapper: vw} c2 := &varWrapperCell{varWrapper: vw} c1.other, c2.other = c2, c1 curCell := c1 _, err := vw.store.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} { rtxn.WithCursor(vw.store.db.Vars, func(cursor *mdbs.Cursor) interface{} { vUUIdBytes, varBytes, err := cursor.Get(nil, nil, mdb.FIRST) if err != nil { cursor.Error(fmt.Errorf("Err on finding first var in %v: %v", vw.store, err)) return nil } if !bytes.Equal(vUUIdBytes, configuration.TopologyVarUUId[:]) { vUUId := common.MakeVarUUId(vUUIdBytes) cursor.Error(fmt.Errorf("Err on finding first var in %v: expected to find topology var, but found %v instead! (%v)", vw.store, vUUId, varBytes)) return nil } for ; err == nil; vUUIdBytes, varBytes, err = cursor.Get(nil, nil, mdb.NEXT) { vUUId := common.MakeVarUUId(vUUIdBytes) seg, _, err := capn.ReadFromMemoryZeroCopy(varBytes) if err != nil { cursor.Error(fmt.Errorf("Err on decoding %v in %v: %v (%v)", vUUId, vw.store, err, varBytes)) return nil } varCap := msgs.ReadRootVar(seg) curCell.vUUId = vUUId curCell.varCap = &varCap vw.c <- curCell curCell = curCell.other } if err != nil && err != mdb.NotFound { cursor.Error(err) } return nil }) return nil }).ResultError() if err != nil { curCell.err = err vw.c <- curCell } }
func ReadTxnFromDisk(rtxn *mdbs.RTxn, txnId *common.TxnId) (*msgs.Txn, error) { bites, err := rtxn.Get(DB.Transactions, txnId[:]) switch err { case nil: if seg, _, err := capn.ReadFromMemoryZeroCopy(bites); err == nil { txn := msgs.ReadRootTxn(seg) return &txn, nil } else { return nil, err } case mdb.NotFound: return nil, nil default: return nil, err } }
func (am *AcceptorManager) loadFromData(txnId *common.TxnId, data []byte) { seg, _, err := capn.ReadFromMemoryZeroCopy(data) if err != nil { log.Println("Unable to decode acceptor state", data) return } state := msgs.ReadRootAcceptorState(seg) txn := state.Txn() instId := instanceId([instanceIdLen]byte{}) instIdSlice := instId[:] outcome := state.Outcome() copy(instIdSlice, txnId[:]) instances := state.Instances() acc := AcceptorFromData(txnId, &txn, &outcome, state.SendToAll(), &instances, am) aInst := &acceptorInstances{acceptor: acc} am.acceptors[*txnId] = aInst for idx, l := 0, instances.Len(); idx < l; idx++ { instancesForVar := instances.At(idx) vUUId := common.MakeVarUUId(instancesForVar.VarId()) acceptedInstances := instancesForVar.Instances() for idy, m := 0, acceptedInstances.Len(); idy < m; idy++ { acceptedInstance := acceptedInstances.At(idy) roundNumber := acceptedInstance.RoundNumber() ballot := acceptedInstance.Ballot() instance := &instance{ manager: am, vUUId: vUUId, promiseNum: paxosNumber(roundNumber), acceptedNum: paxosNumber(roundNumber), accepted: &ballot, } binary.BigEndian.PutUint32(instIdSlice[common.KeyLen:], acceptedInstance.RmId()) copy(instIdSlice[common.KeyLen+4:], vUUId[:]) am.instances[instId] = instance aInst.addInstance(&instId) } } acc.Start() }
func TestV1DataVersioningEmptyToBigger(t *testing.T) { //expOneSet := CapnpEncode("(mylist = [(val = 27),(val = 42)])", "HoldsVerOneDataList") //expOneZeroed := CapnpEncode("(mylist = [(val = 0),(val = 0)])", "HoldsVerOneDataList") //expOneEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerOneDataList") expEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerEmptyList") cv.Convey("Given a struct with 0 data/0 ptr fields, and a newer version of the struct with 1 data fields", t, func() { cv.Convey("then reading from serialized form the small list into the bigger (one or two data values) list should work, getting default value 0 for val/duo.", func() { seg := capn.NewBuffer(nil) scratch := capn.NewBuffer(nil) emptyholder := air.NewRootHoldsVerEmptyList(seg) elist := air.NewVerEmptyList(scratch, 2) emptyholder.SetMylist(elist) actEmpty := ShowSeg(" after NewRootHoldsVerEmptyList(seg) and SetMylist(elist), segment seg is:", seg) actEmptyCap := string(CapnpDecode(actEmpty, "HoldsVerEmptyList")) expEmptyCap := string(CapnpDecode(expEmpty, "HoldsVerEmptyList")) cv.So(actEmptyCap, cv.ShouldResemble, expEmptyCap) fmt.Printf("\n actEmpty is \n") ShowBytes(actEmpty, 10) fmt.Printf("actEmpty decoded by capnp: '%s'\n", string(CapnpDecode(actEmpty, "HoldsVerEmptyList"))) cv.So(actEmpty, cv.ShouldResemble, expEmpty) // seg is set, now read into bigger list buf := bytes.Buffer{} seg.WriteTo(&buf) segbytes := buf.Bytes() reseg, _, err := capn.ReadFromMemoryZeroCopy(segbytes) if err != nil { panic(err) } ShowSeg(" after re-reading segbytes, segment reseg is:", reseg) fmt.Printf("segbytes decoded by capnp as HoldsVerOneDataList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerOneDataList"))) reHolder := air.ReadRootHoldsVerOneDataList(reseg) onelist := reHolder.Mylist() lenone := onelist.Len() cv.So(lenone, cv.ShouldEqual, 2) for i := 0; i < 2; i++ { ele := onelist.At(i) val := ele.Val() cv.So(val, cv.ShouldEqual, 0) } reHolder2 := air.ReadRootHoldsVerTwoDataList(reseg) twolist := reHolder2.Mylist() lentwo := twolist.Len() cv.So(lentwo, cv.ShouldEqual, 2) for i := 0; i < 2; i++ { ele := twolist.At(i) val := ele.Val() cv.So(val, cv.ShouldEqual, 0) duo := ele.Duo() cv.So(duo, cv.ShouldEqual, 0) } }) }) }
func TestDataVersioningZeroPointersToTwo(t *testing.T) { cv.Convey("Given a struct with 2 ptr fields, and another version of the struct with 0 or 1 pointer fields", t, func() { cv.Convey("then reading serialized bigger-struct-list into the smaller (empty or one data-pointer) list should work, truncating/ignoring the new fields", func() { seg := capn.NewBuffer(nil) scratch := capn.NewBuffer(nil) holder := air.NewRootHoldsVerTwoTwoList(seg) twolist := air.NewVerTwoDataTwoPtrList(scratch, 2) plist := capn.PointerList(twolist) d0 := air.NewVerTwoDataTwoPtr(scratch) d0.SetVal(27) d0.SetDuo(26) v1 := air.NewVerOneData(scratch) v1.SetVal(25) v2 := air.NewVerOneData(scratch) v2.SetVal(23) d0.SetPtr1(v1) d0.SetPtr2(v2) d1 := air.NewVerTwoDataTwoPtr(scratch) d1.SetVal(42) d1.SetDuo(41) w1 := air.NewVerOneData(scratch) w1.SetVal(40) w2 := air.NewVerOneData(scratch) w2.SetVal(38) d1.SetPtr1(w1) d1.SetPtr2(w2) plist.Set(0, capn.Object(d0)) plist.Set(1, capn.Object(d1)) holder.SetMylist(twolist) ShowSeg(" before serializing out, segment scratch is:", scratch) ShowSeg(" before serializing out, segment seg is:", seg) // serialize out buf := bytes.Buffer{} seg.WriteTo(&buf) segbytes := buf.Bytes() // and read-back in using smaller expectations reseg, _, err := capn.ReadFromMemoryZeroCopy(segbytes) if err != nil { panic(err) } ShowSeg(" after re-reading segbytes, segment reseg is:", reseg) fmt.Printf("segbytes decoded by capnp as HoldsVerEmptyList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerEmptyList"))) fmt.Printf("segbytes decoded by capnp as HoldsVerOnePtrList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerOnePtrList"))) fmt.Printf("segbytes decoded by capnp as HoldsVerTwoTwoList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerTwoTwoList"))) reHolder := air.ReadRootHoldsVerEmptyList(reseg) elist := reHolder.Mylist() lene := elist.Len() cv.So(lene, cv.ShouldEqual, 2) reHolder1 := air.ReadRootHoldsVerOnePtrList(reseg) onelist := reHolder1.Mylist() lenone := onelist.Len() cv.So(lenone, cv.ShouldEqual, 2) for i := 0; i < 2; i++ { ele := onelist.At(i) ptr1 := ele.Ptr() cv.So(ptr1.Val(), cv.ShouldEqual, twolist.At(i).Ptr1().Val()) } reHolder2 := air.ReadRootHoldsVerTwoTwoPlus(reseg) twolist2 := reHolder2.Mylist() lentwo2 := twolist2.Len() cv.So(lentwo2, cv.ShouldEqual, 2) for i := 0; i < 2; i++ { ele := twolist2.At(i) ptr1 := ele.Ptr1() ptr2 := ele.Ptr2() cv.So(ptr1.Val(), cv.ShouldEqual, twolist.At(i).Ptr1().Val()) //cv.So(ptr1.Duo(), cv.ShouldEqual, twolist.At(i).Ptr1().Duo()) cv.So(ptr2.Val(), cv.ShouldEqual, twolist.At(i).Ptr2().Val()) //cv.So(ptr2.Duo(), cv.ShouldEqual, twolist.At(i).Ptr2().Duo()) cv.So(ele.Tre(), cv.ShouldEqual, 0) cv.So(ele.Lst3().Len(), cv.ShouldEqual, 0) } }) }) }
func (lc *locationChecker) locationCheck(cell *varWrapperCell) error { vUUId := cell.vUUId varCap := cell.varCap foundIn := cell.store fmt.Printf("%v %v\n", foundIn, vUUId) txnId := common.MakeTxnId(varCap.WriteTxnId()) res, err := foundIn.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} { return foundIn.db.ReadTxnBytesFromDisk(rtxn, txnId) }).ResultError() if err != nil { return err } txnBites, ok := res.([]byte) if res == nil || (ok && txnBites == nil) { return fmt.Errorf("Failed to find %v from %v in %v", txnId, vUUId, foundIn) } seg, _, err := capn.ReadFromMemoryZeroCopy(txnBites) if err != nil { return err } txnCap := msgs.ReadRootTxn(seg) positions := varCap.Positions().ToArray() rmIds, err := lc.resolver.ResolveHashCodes(positions) if err != nil { return err } foundLocal := false for _, rmId := range rmIds { if foundLocal = rmId == foundIn.rmId; foundLocal { break } } if !foundLocal { // It must have emigrated but we don't delete. txnId = nil } for _, rmId := range rmIds { if rmId == foundIn.rmId { continue } else if remote, found := lc.stores[rmId]; found { res, err := remote.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} { bites, err := rtxn.Get(remote.db.Vars, vUUId[:]) if err == mdb.NotFound { return nil } else if err == nil { return bites } else { return nil } }).ResultError() if err != nil { return err } varBites, ok := res.([]byte) if res == nil || (ok && varBites == nil) { if vUUId.BootCount() == 1 && vUUId.ConnectionCount() == 0 && (txnId == nil || (txnId.BootCount() == 1 && txnId.ConnectionCount() == 0 && txnCap.Actions().Len() == 1 && txnCap.Actions().At(0).Which() == msgs.ACTION_CREATE)) { fmt.Printf("Failed to find %v in %v (%v, %v, %v) but it looks like it's a bad root.\n", vUUId, remote, rmIds, positions, foundIn) } else { return fmt.Errorf("Failed to find %v in %v (%v, %v, %v)", vUUId, remote, rmIds, positions, foundIn) } } else { seg, _, err := capn.ReadFromMemoryZeroCopy(varBites) if err != nil { return err } remoteTxnId := common.MakeTxnId(msgs.ReadRootVar(seg).WriteTxnId()) if txnId == nil { txnId = remoteTxnId } if remoteTxnId.Compare(txnId) != common.EQ { return fmt.Errorf("%v on %v is at %v; on %v is at %v", vUUId, foundIn, txnId, remote, remoteTxnId) } } } } return nil }
func (s *store) LoadTopology() error { res, err := s.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} { bites, err := rtxn.Get(s.db.Vars, configuration.TopologyVarUUId[:]) if err != nil { rtxn.Error(err) return nil } seg, _, err := capn.ReadFromMemoryZeroCopy(bites) if err != nil { rtxn.Error(err) return nil } varCap := msgs.ReadRootVar(seg) txnId := common.MakeTxnId(varCap.WriteTxnId()) bites = s.db.ReadTxnBytesFromDisk(rtxn, txnId) if bites == nil { rtxn.Error(fmt.Errorf("Unable to find txn for topology: %v", txnId)) return nil } seg, _, err = capn.ReadFromMemoryZeroCopy(bites) if err != nil { rtxn.Error(err) return nil } txnCap := msgs.ReadRootTxn(seg) actions := txnCap.Actions() if actions.Len() != 1 { rtxn.Error(fmt.Errorf("Topology txn has %v actions; expected 1", actions.Len())) return nil } action := actions.At(0) var refs msgs.VarIdPos_List switch action.Which() { case msgs.ACTION_WRITE: w := action.Write() bites = w.Value() refs = w.References() case msgs.ACTION_READWRITE: rw := action.Readwrite() bites = rw.Value() refs = rw.References() case msgs.ACTION_CREATE: c := action.Create() bites = c.Value() refs = c.References() default: rtxn.Error(fmt.Errorf("Expected topology txn action to be w, rw, or c; found %v", action.Which())) return nil } if refs.Len() != 1 { rtxn.Error(fmt.Errorf("Topology txn action has %v references; expected 1", refs.Len())) return nil } rootRef := refs.At(0) seg, _, err = capn.ReadFromMemoryZeroCopy(bites) if err != nil { rtxn.Error(err) return nil } topology, err := configuration.TopologyFromCap(txnId, &rootRef, bites) if err != nil { rtxn.Error(err) return nil } return topology }).ResultError() if err != nil { return err } s.topology = res.(*configuration.Topology) return nil }
// paxos.Connection interface to allow sending to ourself. func (cm *ConnectionManager) Send(b []byte) { seg, _, err := capn.ReadFromMemoryZeroCopy(b) server.CheckFatal(err) msg := msgs.ReadRootMessage(seg) cm.DispatchMessage(cm.RMId, msg.Which(), &msg) }
func TestV1DataVersioningBiggerToEmpty(t *testing.T) { //expTwoSet := CapnpEncode("(mylist = [(val = 27, duo = 26),(val = 42, duo = 41)])", "HoldsVerTwoDataList") //expOneDataOneDefault := CapnpEncode("(mylist = [(val = 27, duo = 0),(val = 42, duo = 0)])", "HoldsVerTwoDataList") //expTwoEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerTwoDataList") //expEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerEmptyList") //expOne := CapnpEncode("(mylist = [(val = 27),(val = 42)])", "HoldsVerOneDataList") cv.Convey("Given a struct with 0 data/0 ptr fields, and a newer version of the struct with 2 data fields", t, func() { cv.Convey("then reading serialized bigger-struct-list into the smaller (empty or one data-member) list should work, truncating/ignoring the new fields", func() { seg := capn.NewBuffer(nil) scratch := capn.NewBuffer(nil) holder := air.NewRootHoldsVerTwoDataList(seg) twolist := air.NewVerTwoDataList(scratch, 2) plist := capn.PointerList(twolist) d0 := air.NewVerTwoData(scratch) d0.SetVal(27) d0.SetDuo(26) d1 := air.NewVerTwoData(scratch) d1.SetVal(42) d1.SetDuo(41) plist.Set(0, capn.Object(d0)) plist.Set(1, capn.Object(d1)) holder.SetMylist(twolist) ShowSeg(" before serializing out, segment scratch is:", scratch) ShowSeg(" before serializing out, segment seg is:", seg) // serialize out buf := bytes.Buffer{} seg.WriteTo(&buf) segbytes := buf.Bytes() // and read-back in using smaller expectations reseg, _, err := capn.ReadFromMemoryZeroCopy(segbytes) if err != nil { panic(err) } ShowSeg(" after re-reading segbytes, segment reseg is:", reseg) fmt.Printf("segbytes decoded by capnp as HoldsVerEmptyList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerEmptyList"))) fmt.Printf("segbytes decoded by capnp as HoldsVerOneDataList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerOneDataList"))) fmt.Printf("segbytes decoded by capnp as HoldsVerTwoDataList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerTwoDataList"))) reHolder := air.ReadRootHoldsVerEmptyList(reseg) elist := reHolder.Mylist() lene := elist.Len() cv.So(lene, cv.ShouldEqual, 2) reHolder1 := air.ReadRootHoldsVerOneDataList(reseg) onelist := reHolder1.Mylist() lenone := onelist.Len() cv.So(lenone, cv.ShouldEqual, 2) for i := 0; i < 2; i++ { ele := onelist.At(i) val := ele.Val() cv.So(val, cv.ShouldEqual, twolist.At(i).Val()) } reHolder2 := air.ReadRootHoldsVerTwoDataList(reseg) twolist2 := reHolder2.Mylist() lentwo2 := twolist2.Len() cv.So(lentwo2, cv.ShouldEqual, 2) for i := 0; i < 2; i++ { ele := twolist2.At(i) val := ele.Val() duo := ele.Duo() cv.So(val, cv.ShouldEqual, twolist.At(i).Val()) cv.So(duo, cv.ShouldEqual, twolist.At(i).Duo()) } }) }) }
func TestDataVersioningAvoidsUnnecessaryTruncation(t *testing.T) { expFull := CapnpEncode("(val = 9, duo = 8, ptr1 = (val = 77), ptr2 = (val = 55))", "VerTwoDataTwoPtr") //expEmpty := CapnpEncode("()", "VerEmpty") cv.Convey("Given a struct with 0 ptr fields, and a newer version of the struct with two data and two pointer fields", t, func() { cv.Convey("then old code expecting the smaller struct but reading the newer-bigger struct should not truncate it if it doesn't have to (e.g. not assigning into a composite list), and should preserve all data when re-serializing it.", func() { seg := capn.NewBuffer(nil) scratch := capn.NewBuffer(nil) big := air.NewRootVerTwoDataTwoPtr(seg) one := air.NewVerOneData(scratch) one.SetVal(77) two := air.NewVerOneData(scratch) two.SetVal(55) big.SetVal(9) big.SetDuo(8) big.SetPtr1(one) big.SetPtr2(two) bigVerBytes := ShowSeg("\n\n with our 2x2 new big struct, segment seg is:", seg) cv.So(bigVerBytes, cv.ShouldResemble, expFull) // now pretend to be an old client, reading and writing // expecting an empty struct, but full data should be preserved // and written, because we aren't writing into a cramped/ // fixed-space composite-list space. // Before test, verify that if we force reading into text-form, we get // what we expect. actEmptyCap := string(CapnpDecode(bigVerBytes, "VerEmpty")) cv.So(actEmptyCap, cv.ShouldResemble, "()\n") // okay, now the actual test: weThinkEmptyButActuallyFull := air.ReadRootVerEmpty(seg) freshSeg := capn.NewBuffer(nil) wrapEmpty := air.NewRootWrapEmpty(freshSeg) // here is the critical step, this should not truncate: wrapEmpty.SetMightNotBeReallyEmpty(weThinkEmptyButActuallyFull) // now verify: freshBytes := ShowSeg("\n\n after wrapEmpty.SetMightNotBeReallyEmpty(weThinkEmptyButActuallyFull), segment freshSeg is:", freshSeg) reseg, _, err := capn.ReadFromMemoryZeroCopy(freshBytes) if err != nil { panic(err) } ShowSeg(" after re-reading freshBytes, segment reseg is:", reseg) fmt.Printf("freshBytes decoded by capnp as Wrap2x2: '%s'\n", string(CapnpDecode(freshBytes, "Wrap2x2"))) wrap22 := air.ReadRootWrap2x2plus(reseg) notEmpty := wrap22.MightNotBeReallyEmpty() val := notEmpty.Val() cv.So(val, cv.ShouldEqual, 9) duo := notEmpty.Duo() cv.So(duo, cv.ShouldEqual, 8) ptr1 := notEmpty.Ptr1() ptr2 := notEmpty.Ptr2() cv.So(ptr1.Val(), cv.ShouldEqual, 77) cv.So(ptr2.Val(), cv.ShouldEqual, 55) // Tre should get the default, as it was never set cv.So(notEmpty.Tre(), cv.ShouldEqual, 0) // same for Lst3 cv.So(notEmpty.Lst3().Len(), cv.ShouldEqual, 0) }) }) }