func Test002ListListStructList(t *testing.T) {
cv.Convey("Given type RWTest struct { NestMatrix [][]Nester1; } in go, where Nester1 is a struct, and a mirror/parallel capnp struct air.RWTestCapn { nestMatrix @0: List(List(Nester1Capn)); } defined in the aircraftlib schema", t, func() {
cv.Convey("When we Save() RWTest to capn and then Load() it back, the data should match, so that we have working List(List(Struct)) serialization and deserializatoin in go-capnproto", func() {
// full RWTest
rw := RWTest{
NestMatrix: [][]Nester1{[]Nester1{Nester1{Strs: []string{"z", "w"}}, Nester1{Strs: []string{"q", "r"}}}, []Nester1{Nester1{Strs: []string{"zebra", "wally"}}, Nester1{Strs: []string{"qubert", "rocks"}}}},
}
var o bytes.Buffer
rw.Save(&o)
seg, n, err := capn.ReadFromMemoryZeroCopy(o.Bytes())
cv.So(err, cv.ShouldEqual, nil)
cv.So(n, cv.ShouldBeGreaterThan, 0)
text := CapnpDecodeSegment(seg, "", "aircraftlib/aircraft.capnp", "RWTestCapn")
if false {
fmt.Printf("text = '%s'\n", text)
}
rw2 := &RWTest{}
rw2.Load(&o)
//fmt.Printf("rw = '%#v'\n", rw)
//fmt.Printf("rw2 = '%#v'\n", rw2)
same := reflect.DeepEqual(&rw, rw2)
cv.So(same, cv.ShouldEqual, true)
})
})
}
// start with smaller Struct(List)
func Test001StructList(t *testing.T) {
cv.Convey("Given type Nester1 struct { Strs []string } in go, where Nester1 is a struct, and a mirror/parallel capnp struct air.Nester1Capn { strs @0: List(Text); } defined in the aircraftlib schema", t, func() {
cv.Convey("When we Save() Nester to capn and then Load() it back, the data should match, so that we have working Struct(List) serialization and deserializatoin in go-capnproto", func() {
// Does Nester1 alone serialization and deser okay?
rw := Nester1{Strs: []string{"xenophilia", "watchowski"}}
var o bytes.Buffer
rw.Save(&o)
seg, n, err := capn.ReadFromMemoryZeroCopy(o.Bytes())
cv.So(err, cv.ShouldEqual, nil)
cv.So(n, cv.ShouldBeGreaterThan, 0)
text := CapnpDecodeSegment(seg, "", "aircraftlib/aircraft.capnp", "Nester1Capn")
if false {
fmt.Printf("text = '%s'\n", text)
}
rw2 := &Nester1{}
rw2.Load(&o)
//fmt.Printf("rw = '%#v'\n", rw)
//fmt.Printf("rw2 = '%#v'\n", rw2)
same := reflect.DeepEqual(&rw, rw2)
cv.So(same, cv.ShouldEqual, true)
})
})
}
// disk file of a capn segment -> in-memory capn segment -> stdin to capnp decode -> stdout human-readble string form
func CapnFileToText(serializedCapnpFilePathToDisplay string, capnpSchemaFilePath string, capnpExePath string) (string, error) {
// a) read file into Segment
byteslice, err := ioutil.ReadFile(serializedCapnpFilePathToDisplay)
if err != nil {
return "", err
}
seg, nbytes, err := capn.ReadFromMemoryZeroCopy(byteslice)
if err == io.EOF {
return "", err
}
if err != nil {
return "", err
}
if nbytes == 0 {
return "", errors.New(fmt.Sprintf("did not expect 0 bytes back from capn.ReadFromMemoryZeroCopy() on reading file '%s'", serializedCapnpFilePathToDisplay))
}
// b) tell CapnpDecodeSegment() to show the human-readable-text form of the message
// warning: CapnpDecodeSegment() may panic on you. It is a testing utility so that
// is desirable. For production, do something else.
return CapnpDecodeSegment(seg, capnpExePath, capnpSchemaFilePath, "Z"), nil
}
func ProposerFromData(pm *ProposerManager, txnId *common.TxnId, data []byte) *Proposer {
seg, _, err := capn.ReadFromMemoryZeroCopy(data)
if err != nil {
log.Println("Unable to decode proposer state", data)
}
// If we were on disk, then that means we must be locally complete
// and just need to send out TLCs.
state := msgs.ReadRootProposerState(seg)
acceptorsCap := state.Acceptors()
acceptors := make([]common.RMId, acceptorsCap.Len())
for idx := range acceptors {
acceptors[idx] = common.RMId(acceptorsCap.At(idx))
}
// We were on disk. Thus we received outcomes from all
// acceptors. So we don't need to worry about the outcome
// accumulator's fInc, hence just use -1 here.
p := &Proposer{
proposerManager: pm,
mode: proposerTLCSender,
txnId: txnId,
acceptors: acceptors,
fInc: -1,
}
p.init()
p.allAcceptorsAgreed = true
return p
}
func VarFromData(data []byte, exe *dispatcher.Executor, disk *mdbs.MDBServer, vm *VarManager) (*Var, error) {
seg, _, err := capn.ReadFromMemoryZeroCopy(data)
if err != nil {
return nil, err
}
varCap := msgs.ReadRootVar(seg)
v := newVar(common.MakeVarUUId(varCap.Id()), exe, disk, vm)
positions := varCap.Positions()
if positions.Len() != 0 {
v.positions = (*common.Positions)(&positions)
}
writeTxnId := common.MakeTxnId(varCap.WriteTxnId())
writeTxnClock := VectorClockFromCap(varCap.WriteTxnClock())
writesClock := VectorClockFromCap(varCap.WritesClock())
server.Log(v.UUId, "Restored", writeTxnId)
if result, err := disk.ReadonlyTransaction(func(rtxn *mdbs.RTxn) (interface{}, error) {
return db.ReadTxnFromDisk(rtxn, writeTxnId)
}).ResultError(); err == nil {
if result == nil || result.(*msgs.Txn) == nil {
panic(fmt.Sprintf("%v Unable to find txn %v on disk (%v)", v.UUId, writeTxnId, result))
}
actions := result.(*msgs.Txn).Actions()
v.curFrame = NewFrame(nil, v, writeTxnId, &actions, writeTxnClock, writesClock)
v.curFrameOnDisk = v.curFrame
} else {
return nil, err
}
v.varCap = &varCap
return v, nil
}
func (cah *connectionAwaitHandshake) readAndDecryptOne() (*capn.Segment, error) {
if cah.sessionKey == nil {
return capn.ReadFromStream(cah.socket, nil)
}
read, err := cah.socket.Read(cah.inBuff)
if err != nil {
return nil, err
} else if read < len(cah.inBuff) {
return nil, fmt.Errorf("Only read %v bytes, wanted %v", read, len(cah.inBuff))
}
copy(cah.nonceAryIn[16:], cah.inBuff[:8])
msgLen := binary.BigEndian.Uint64(cah.inBuff[8:16])
plainLen := msgLen - secretbox.Overhead
msgBuf := make([]byte, plainLen+msgLen)
for recvBuf := msgBuf[plainLen:]; len(recvBuf) != 0; {
read, err = cah.socket.Read(recvBuf)
if err != nil {
return nil, err
} else {
recvBuf = recvBuf[read:]
}
}
plaintext, ok := secretbox.Open(msgBuf[:0], msgBuf[plainLen:], cah.nonceAryIn, cah.sessionKey)
if !ok {
return nil, fmt.Errorf("Unable to decrypt message")
}
seg, _, err := capn.ReadFromMemoryZeroCopy(plaintext)
return seg, err
}
func TopologyDeserialize(txnId *common.TxnId, root *msgs.VarIdPos, data []byte) (*Topology, error) {
seg, _, err := capn.ReadFromMemoryZeroCopy(data)
if err != nil {
return nil, err
}
topology := msgs.ReadRootTopology(seg)
return TopologyFromCap(txnId, root, &topology), nil
}
func TopologyFromCap(txnId *common.TxnId, root *msgs.VarIdPos, data []byte) (*Topology, error) {
seg, _, err := capn.ReadFromMemoryZeroCopy(data)
if err != nil {
return nil, err
}
configCap := msgs.ReadRootConfiguration(seg)
config := ConfigurationFromCap(&configCap)
return NewTopology(txnId, root, config), nil
}
func TestDataVersioningZeroPointersToMore(t *testing.T) {
expEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerEmptyList")
cv.Convey("Given a struct with 0 ptr fields, and a newer version of the struct with 1-2 pointer fields", t, func() {
cv.Convey("then serializing the empty list and reading it back into 1 or 2 pointer fields should default initialize the pointer fields", func() {
seg := capn.NewBuffer(nil)
scratch := capn.NewBuffer(nil)
emptyholder := air.NewRootHoldsVerEmptyList(seg)
elist := air.NewVerEmptyList(scratch, 2)
emptyholder.SetMylist(elist)
actEmpty := ShowSeg(" after NewRootHoldsVerEmptyList(seg) and SetMylist(elist), segment seg is:", seg)
actEmptyCap := string(CapnpDecode(actEmpty, "HoldsVerEmptyList"))
expEmptyCap := string(CapnpDecode(expEmpty, "HoldsVerEmptyList"))
cv.So(actEmptyCap, cv.ShouldResemble, expEmptyCap)
fmt.Printf("\n actEmpty is \n")
ShowBytes(actEmpty, 10)
fmt.Printf("actEmpty decoded by capnp: '%s'\n", string(CapnpDecode(actEmpty, "HoldsVerEmptyList")))
cv.So(actEmpty, cv.ShouldResemble, expEmpty)
// seg is set, now read into bigger list
buf := bytes.Buffer{}
seg.WriteTo(&buf)
segbytes := buf.Bytes()
reseg, _, err := capn.ReadFromMemoryZeroCopy(segbytes)
if err != nil {
panic(err)
}
ShowSeg(" after re-reading segbytes, segment reseg is:", reseg)
fmt.Printf("segbytes decoded by capnp as HoldsVerOneDataList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerOneDataList")))
reHolder := air.ReadRootHoldsVerTwoTwoList(reseg)
list22 := reHolder.Mylist()
len22 := list22.Len()
cv.So(len22, cv.ShouldEqual, 2)
for i := 0; i < 2; i++ {
ele := list22.At(i)
val := ele.Val()
cv.So(val, cv.ShouldEqual, 0)
duo := ele.Duo()
cv.So(duo, cv.ShouldEqual, 0)
ptr1 := ele.Ptr1()
ptr2 := ele.Ptr2()
fmt.Printf("ptr1 = %#v\n", ptr1)
cv.So(ptr1.Segment, cv.ShouldEqual, nil)
fmt.Printf("ptr2 = %#v\n", ptr2)
cv.So(ptr2.Segment, cv.ShouldEqual, nil)
}
})
})
}
func VarFromData(data []byte, exe *dispatcher.Executor, db *db.Databases, vm *VarManager) (*Var, error) {
seg, _, err := capn.ReadFromMemoryZeroCopy(data)
if err != nil {
return nil, err
}
varCap := msgs.ReadRootVar(seg)
v := newVar(common.MakeVarUUId(varCap.Id()), exe, db, vm)
positions := varCap.Positions()
if positions.Len() != 0 {
v.positions = (*common.Positions)(&positions)
}
writeTxnId := common.MakeTxnId(varCap.WriteTxnId())
writeTxnClock := VectorClockFromCap(varCap.WriteTxnClock())
writesClock := VectorClockFromCap(varCap.WritesClock())
server.Log(v.UUId, "Restored", writeTxnId)
if result, err := db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
return db.ReadTxnBytesFromDisk(rtxn, writeTxnId)
}).ResultError(); err == nil && result != nil {
bites := result.([]byte)
if seg, _, err := capn.ReadFromMemoryZeroCopy(bites); err == nil {
txn := msgs.ReadRootTxn(seg)
actions := txn.Actions()
v.curFrame = NewFrame(nil, v, writeTxnId, &actions, writeTxnClock, writesClock)
v.curFrameOnDisk = v.curFrame
v.varCap = &varCap
return v, nil
} else {
return nil, err
}
} else {
return nil, err
}
}
func loadVars(disk *mdbs.MDBServer, vars map[common.VarUUId]*varstate) {
_, err := disk.ReadonlyTransaction(func(rtxn *mdbs.RTxn) (interface{}, error) {
return rtxn.WithCursor(db.DB.Vars, func(cursor *mdb.Cursor) (interface{}, error) {
key, data, err := cursor.Get(nil, nil, mdb.FIRST)
for ; err == nil; key, data, err = cursor.Get(nil, nil, mdb.NEXT) {
vUUId := common.MakeVarUUId(key)
seg, _, err := capn.ReadFromMemoryZeroCopy(data)
if err != nil {
log.Println(err)
continue
}
varCap := msgs.ReadRootVar(seg)
pos := varCap.Positions()
positions := (*common.Positions)(&pos)
writeTxnId := common.MakeTxnId(varCap.WriteTxnId())
writeTxnClock := eng.VectorClockFromCap(varCap.WriteTxnClock())
writesClock := eng.VectorClockFromCap(varCap.WritesClock())
if state, found := vars[*vUUId]; found {
if err := state.matches(disk, writeTxnId, writeTxnClock, writesClock, positions); err != nil {
log.Println(err)
}
} else {
state = &varstate{
vUUId: vUUId,
disks: []*mdbs.MDBServer{disk},
writeTxnId: writeTxnId,
writeTxnClock: writeTxnClock,
writeWritesClock: writesClock,
positions: positions,
}
vars[*vUUId] = state
}
}
if err == mdb.NotFound {
return nil, nil
} else {
return nil, err
}
})
}).ResultError()
if err != nil {
log.Println(err)
}
}
func (vw *varWrapper) start() {
defer close(vw.c)
c1 := &varWrapperCell{varWrapper: vw}
c2 := &varWrapperCell{varWrapper: vw}
c1.other, c2.other = c2, c1
curCell := c1
_, err := vw.store.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
rtxn.WithCursor(vw.store.db.Vars, func(cursor *mdbs.Cursor) interface{} {
vUUIdBytes, varBytes, err := cursor.Get(nil, nil, mdb.FIRST)
if err != nil {
cursor.Error(fmt.Errorf("Err on finding first var in %v: %v", vw.store, err))
return nil
}
if !bytes.Equal(vUUIdBytes, configuration.TopologyVarUUId[:]) {
vUUId := common.MakeVarUUId(vUUIdBytes)
cursor.Error(fmt.Errorf("Err on finding first var in %v: expected to find topology var, but found %v instead! (%v)", vw.store, vUUId, varBytes))
return nil
}
for ; err == nil; vUUIdBytes, varBytes, err = cursor.Get(nil, nil, mdb.NEXT) {
vUUId := common.MakeVarUUId(vUUIdBytes)
seg, _, err := capn.ReadFromMemoryZeroCopy(varBytes)
if err != nil {
cursor.Error(fmt.Errorf("Err on decoding %v in %v: %v (%v)", vUUId, vw.store, err, varBytes))
return nil
}
varCap := msgs.ReadRootVar(seg)
curCell.vUUId = vUUId
curCell.varCap = &varCap
vw.c <- curCell
curCell = curCell.other
}
if err != nil && err != mdb.NotFound {
cursor.Error(err)
}
return nil
})
return nil
}).ResultError()
if err != nil {
curCell.err = err
vw.c <- curCell
}
}
func ReadTxnFromDisk(rtxn *mdbs.RTxn, txnId *common.TxnId) (*msgs.Txn, error) {
bites, err := rtxn.Get(DB.Transactions, txnId[:])
switch err {
case nil:
if seg, _, err := capn.ReadFromMemoryZeroCopy(bites); err == nil {
txn := msgs.ReadRootTxn(seg)
return &txn, nil
} else {
return nil, err
}
case mdb.NotFound:
return nil, nil
default:
return nil, err
}
}
func (am *AcceptorManager) loadFromData(txnId *common.TxnId, data []byte) {
seg, _, err := capn.ReadFromMemoryZeroCopy(data)
if err != nil {
log.Println("Unable to decode acceptor state", data)
return
}
state := msgs.ReadRootAcceptorState(seg)
txn := state.Txn()
instId := instanceId([instanceIdLen]byte{})
instIdSlice := instId[:]
outcome := state.Outcome()
copy(instIdSlice, txnId[:])
instances := state.Instances()
acc := AcceptorFromData(txnId, &txn, &outcome, state.SendToAll(), &instances, am)
aInst := &acceptorInstances{acceptor: acc}
am.acceptors[*txnId] = aInst
for idx, l := 0, instances.Len(); idx < l; idx++ {
instancesForVar := instances.At(idx)
vUUId := common.MakeVarUUId(instancesForVar.VarId())
acceptedInstances := instancesForVar.Instances()
for idy, m := 0, acceptedInstances.Len(); idy < m; idy++ {
acceptedInstance := acceptedInstances.At(idy)
roundNumber := acceptedInstance.RoundNumber()
ballot := acceptedInstance.Ballot()
instance := &instance{
manager: am,
vUUId: vUUId,
promiseNum: paxosNumber(roundNumber),
acceptedNum: paxosNumber(roundNumber),
accepted: &ballot,
}
binary.BigEndian.PutUint32(instIdSlice[common.KeyLen:], acceptedInstance.RmId())
copy(instIdSlice[common.KeyLen+4:], vUUId[:])
am.instances[instId] = instance
aInst.addInstance(&instId)
}
}
acc.Start()
}
func TestV1DataVersioningEmptyToBigger(t *testing.T) {
//expOneSet := CapnpEncode("(mylist = [(val = 27),(val = 42)])", "HoldsVerOneDataList")
//expOneZeroed := CapnpEncode("(mylist = [(val = 0),(val = 0)])", "HoldsVerOneDataList")
//expOneEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerOneDataList")
expEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerEmptyList")
cv.Convey("Given a struct with 0 data/0 ptr fields, and a newer version of the struct with 1 data fields", t, func() {
cv.Convey("then reading from serialized form the small list into the bigger (one or two data values) list should work, getting default value 0 for val/duo.", func() {
seg := capn.NewBuffer(nil)
scratch := capn.NewBuffer(nil)
emptyholder := air.NewRootHoldsVerEmptyList(seg)
elist := air.NewVerEmptyList(scratch, 2)
emptyholder.SetMylist(elist)
actEmpty := ShowSeg(" after NewRootHoldsVerEmptyList(seg) and SetMylist(elist), segment seg is:", seg)
actEmptyCap := string(CapnpDecode(actEmpty, "HoldsVerEmptyList"))
expEmptyCap := string(CapnpDecode(expEmpty, "HoldsVerEmptyList"))
cv.So(actEmptyCap, cv.ShouldResemble, expEmptyCap)
fmt.Printf("\n actEmpty is \n")
ShowBytes(actEmpty, 10)
fmt.Printf("actEmpty decoded by capnp: '%s'\n", string(CapnpDecode(actEmpty, "HoldsVerEmptyList")))
cv.So(actEmpty, cv.ShouldResemble, expEmpty)
// seg is set, now read into bigger list
buf := bytes.Buffer{}
seg.WriteTo(&buf)
segbytes := buf.Bytes()
reseg, _, err := capn.ReadFromMemoryZeroCopy(segbytes)
if err != nil {
panic(err)
}
ShowSeg(" after re-reading segbytes, segment reseg is:", reseg)
fmt.Printf("segbytes decoded by capnp as HoldsVerOneDataList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerOneDataList")))
reHolder := air.ReadRootHoldsVerOneDataList(reseg)
onelist := reHolder.Mylist()
lenone := onelist.Len()
cv.So(lenone, cv.ShouldEqual, 2)
for i := 0; i < 2; i++ {
ele := onelist.At(i)
val := ele.Val()
cv.So(val, cv.ShouldEqual, 0)
}
reHolder2 := air.ReadRootHoldsVerTwoDataList(reseg)
twolist := reHolder2.Mylist()
lentwo := twolist.Len()
cv.So(lentwo, cv.ShouldEqual, 2)
for i := 0; i < 2; i++ {
ele := twolist.At(i)
val := ele.Val()
cv.So(val, cv.ShouldEqual, 0)
duo := ele.Duo()
cv.So(duo, cv.ShouldEqual, 0)
}
})
})
}
func TestDataVersioningZeroPointersToTwo(t *testing.T) {
cv.Convey("Given a struct with 2 ptr fields, and another version of the struct with 0 or 1 pointer fields", t, func() {
cv.Convey("then reading serialized bigger-struct-list into the smaller (empty or one data-pointer) list should work, truncating/ignoring the new fields", func() {
seg := capn.NewBuffer(nil)
scratch := capn.NewBuffer(nil)
holder := air.NewRootHoldsVerTwoTwoList(seg)
twolist := air.NewVerTwoDataTwoPtrList(scratch, 2)
plist := capn.PointerList(twolist)
d0 := air.NewVerTwoDataTwoPtr(scratch)
d0.SetVal(27)
d0.SetDuo(26)
v1 := air.NewVerOneData(scratch)
v1.SetVal(25)
v2 := air.NewVerOneData(scratch)
v2.SetVal(23)
d0.SetPtr1(v1)
d0.SetPtr2(v2)
d1 := air.NewVerTwoDataTwoPtr(scratch)
d1.SetVal(42)
d1.SetDuo(41)
w1 := air.NewVerOneData(scratch)
w1.SetVal(40)
w2 := air.NewVerOneData(scratch)
w2.SetVal(38)
d1.SetPtr1(w1)
d1.SetPtr2(w2)
plist.Set(0, capn.Object(d0))
plist.Set(1, capn.Object(d1))
holder.SetMylist(twolist)
ShowSeg(" before serializing out, segment scratch is:", scratch)
ShowSeg(" before serializing out, segment seg is:", seg)
// serialize out
buf := bytes.Buffer{}
seg.WriteTo(&buf)
segbytes := buf.Bytes()
// and read-back in using smaller expectations
reseg, _, err := capn.ReadFromMemoryZeroCopy(segbytes)
if err != nil {
panic(err)
}
ShowSeg(" after re-reading segbytes, segment reseg is:", reseg)
fmt.Printf("segbytes decoded by capnp as HoldsVerEmptyList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerEmptyList")))
fmt.Printf("segbytes decoded by capnp as HoldsVerOnePtrList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerOnePtrList")))
fmt.Printf("segbytes decoded by capnp as HoldsVerTwoTwoList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerTwoTwoList")))
reHolder := air.ReadRootHoldsVerEmptyList(reseg)
elist := reHolder.Mylist()
lene := elist.Len()
cv.So(lene, cv.ShouldEqual, 2)
reHolder1 := air.ReadRootHoldsVerOnePtrList(reseg)
onelist := reHolder1.Mylist()
lenone := onelist.Len()
cv.So(lenone, cv.ShouldEqual, 2)
for i := 0; i < 2; i++ {
ele := onelist.At(i)
ptr1 := ele.Ptr()
cv.So(ptr1.Val(), cv.ShouldEqual, twolist.At(i).Ptr1().Val())
}
reHolder2 := air.ReadRootHoldsVerTwoTwoPlus(reseg)
twolist2 := reHolder2.Mylist()
lentwo2 := twolist2.Len()
cv.So(lentwo2, cv.ShouldEqual, 2)
for i := 0; i < 2; i++ {
ele := twolist2.At(i)
ptr1 := ele.Ptr1()
ptr2 := ele.Ptr2()
cv.So(ptr1.Val(), cv.ShouldEqual, twolist.At(i).Ptr1().Val())
//cv.So(ptr1.Duo(), cv.ShouldEqual, twolist.At(i).Ptr1().Duo())
cv.So(ptr2.Val(), cv.ShouldEqual, twolist.At(i).Ptr2().Val())
//cv.So(ptr2.Duo(), cv.ShouldEqual, twolist.At(i).Ptr2().Duo())
cv.So(ele.Tre(), cv.ShouldEqual, 0)
cv.So(ele.Lst3().Len(), cv.ShouldEqual, 0)
}
})
})
}
func (lc *locationChecker) locationCheck(cell *varWrapperCell) error {
vUUId := cell.vUUId
varCap := cell.varCap
foundIn := cell.store
fmt.Printf("%v %v\n", foundIn, vUUId)
txnId := common.MakeTxnId(varCap.WriteTxnId())
res, err := foundIn.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
return foundIn.db.ReadTxnBytesFromDisk(rtxn, txnId)
}).ResultError()
if err != nil {
return err
}
txnBites, ok := res.([]byte)
if res == nil || (ok && txnBites == nil) {
return fmt.Errorf("Failed to find %v from %v in %v", txnId, vUUId, foundIn)
}
seg, _, err := capn.ReadFromMemoryZeroCopy(txnBites)
if err != nil {
return err
}
txnCap := msgs.ReadRootTxn(seg)
positions := varCap.Positions().ToArray()
rmIds, err := lc.resolver.ResolveHashCodes(positions)
if err != nil {
return err
}
foundLocal := false
for _, rmId := range rmIds {
if foundLocal = rmId == foundIn.rmId; foundLocal {
break
}
}
if !foundLocal {
// It must have emigrated but we don't delete.
txnId = nil
}
for _, rmId := range rmIds {
if rmId == foundIn.rmId {
continue
} else if remote, found := lc.stores[rmId]; found {
res, err := remote.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
bites, err := rtxn.Get(remote.db.Vars, vUUId[:])
if err == mdb.NotFound {
return nil
} else if err == nil {
return bites
} else {
return nil
}
}).ResultError()
if err != nil {
return err
}
varBites, ok := res.([]byte)
if res == nil || (ok && varBites == nil) {
if vUUId.BootCount() == 1 && vUUId.ConnectionCount() == 0 &&
(txnId == nil ||
(txnId.BootCount() == 1 && txnId.ConnectionCount() == 0 &&
txnCap.Actions().Len() == 1 && txnCap.Actions().At(0).Which() == msgs.ACTION_CREATE)) {
fmt.Printf("Failed to find %v in %v (%v, %v, %v) but it looks like it's a bad root.\n", vUUId, remote, rmIds, positions, foundIn)
} else {
return fmt.Errorf("Failed to find %v in %v (%v, %v, %v)", vUUId, remote, rmIds, positions, foundIn)
}
} else {
seg, _, err := capn.ReadFromMemoryZeroCopy(varBites)
if err != nil {
return err
}
remoteTxnId := common.MakeTxnId(msgs.ReadRootVar(seg).WriteTxnId())
if txnId == nil {
txnId = remoteTxnId
}
if remoteTxnId.Compare(txnId) != common.EQ {
return fmt.Errorf("%v on %v is at %v; on %v is at %v", vUUId, foundIn, txnId, remote, remoteTxnId)
}
}
}
}
return nil
}
func (s *store) LoadTopology() error {
res, err := s.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
bites, err := rtxn.Get(s.db.Vars, configuration.TopologyVarUUId[:])
if err != nil {
rtxn.Error(err)
return nil
}
seg, _, err := capn.ReadFromMemoryZeroCopy(bites)
if err != nil {
rtxn.Error(err)
return nil
}
varCap := msgs.ReadRootVar(seg)
txnId := common.MakeTxnId(varCap.WriteTxnId())
bites = s.db.ReadTxnBytesFromDisk(rtxn, txnId)
if bites == nil {
rtxn.Error(fmt.Errorf("Unable to find txn for topology: %v", txnId))
return nil
}
seg, _, err = capn.ReadFromMemoryZeroCopy(bites)
if err != nil {
rtxn.Error(err)
return nil
}
txnCap := msgs.ReadRootTxn(seg)
actions := txnCap.Actions()
if actions.Len() != 1 {
rtxn.Error(fmt.Errorf("Topology txn has %v actions; expected 1", actions.Len()))
return nil
}
action := actions.At(0)
var refs msgs.VarIdPos_List
switch action.Which() {
case msgs.ACTION_WRITE:
w := action.Write()
bites = w.Value()
refs = w.References()
case msgs.ACTION_READWRITE:
rw := action.Readwrite()
bites = rw.Value()
refs = rw.References()
case msgs.ACTION_CREATE:
c := action.Create()
bites = c.Value()
refs = c.References()
default:
rtxn.Error(fmt.Errorf("Expected topology txn action to be w, rw, or c; found %v", action.Which()))
return nil
}
if refs.Len() != 1 {
rtxn.Error(fmt.Errorf("Topology txn action has %v references; expected 1", refs.Len()))
return nil
}
rootRef := refs.At(0)
seg, _, err = capn.ReadFromMemoryZeroCopy(bites)
if err != nil {
rtxn.Error(err)
return nil
}
topology, err := configuration.TopologyFromCap(txnId, &rootRef, bites)
if err != nil {
rtxn.Error(err)
return nil
}
return topology
}).ResultError()
if err != nil {
return err
}
s.topology = res.(*configuration.Topology)
return nil
}
// paxos.Connection interface to allow sending to ourself.
func (cm *ConnectionManager) Send(b []byte) {
seg, _, err := capn.ReadFromMemoryZeroCopy(b)
server.CheckFatal(err)
msg := msgs.ReadRootMessage(seg)
cm.DispatchMessage(cm.RMId, msg.Which(), &msg)
}
func TestV1DataVersioningBiggerToEmpty(t *testing.T) {
//expTwoSet := CapnpEncode("(mylist = [(val = 27, duo = 26),(val = 42, duo = 41)])", "HoldsVerTwoDataList")
//expOneDataOneDefault := CapnpEncode("(mylist = [(val = 27, duo = 0),(val = 42, duo = 0)])", "HoldsVerTwoDataList")
//expTwoEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerTwoDataList")
//expEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerEmptyList")
//expOne := CapnpEncode("(mylist = [(val = 27),(val = 42)])", "HoldsVerOneDataList")
cv.Convey("Given a struct with 0 data/0 ptr fields, and a newer version of the struct with 2 data fields", t, func() {
cv.Convey("then reading serialized bigger-struct-list into the smaller (empty or one data-member) list should work, truncating/ignoring the new fields", func() {
seg := capn.NewBuffer(nil)
scratch := capn.NewBuffer(nil)
holder := air.NewRootHoldsVerTwoDataList(seg)
twolist := air.NewVerTwoDataList(scratch, 2)
plist := capn.PointerList(twolist)
d0 := air.NewVerTwoData(scratch)
d0.SetVal(27)
d0.SetDuo(26)
d1 := air.NewVerTwoData(scratch)
d1.SetVal(42)
d1.SetDuo(41)
plist.Set(0, capn.Object(d0))
plist.Set(1, capn.Object(d1))
holder.SetMylist(twolist)
ShowSeg(" before serializing out, segment scratch is:", scratch)
ShowSeg(" before serializing out, segment seg is:", seg)
// serialize out
buf := bytes.Buffer{}
seg.WriteTo(&buf)
segbytes := buf.Bytes()
// and read-back in using smaller expectations
reseg, _, err := capn.ReadFromMemoryZeroCopy(segbytes)
if err != nil {
panic(err)
}
ShowSeg(" after re-reading segbytes, segment reseg is:", reseg)
fmt.Printf("segbytes decoded by capnp as HoldsVerEmptyList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerEmptyList")))
fmt.Printf("segbytes decoded by capnp as HoldsVerOneDataList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerOneDataList")))
fmt.Printf("segbytes decoded by capnp as HoldsVerTwoDataList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerTwoDataList")))
reHolder := air.ReadRootHoldsVerEmptyList(reseg)
elist := reHolder.Mylist()
lene := elist.Len()
cv.So(lene, cv.ShouldEqual, 2)
reHolder1 := air.ReadRootHoldsVerOneDataList(reseg)
onelist := reHolder1.Mylist()
lenone := onelist.Len()
cv.So(lenone, cv.ShouldEqual, 2)
for i := 0; i < 2; i++ {
ele := onelist.At(i)
val := ele.Val()
cv.So(val, cv.ShouldEqual, twolist.At(i).Val())
}
reHolder2 := air.ReadRootHoldsVerTwoDataList(reseg)
twolist2 := reHolder2.Mylist()
lentwo2 := twolist2.Len()
cv.So(lentwo2, cv.ShouldEqual, 2)
for i := 0; i < 2; i++ {
ele := twolist2.At(i)
val := ele.Val()
duo := ele.Duo()
cv.So(val, cv.ShouldEqual, twolist.At(i).Val())
cv.So(duo, cv.ShouldEqual, twolist.At(i).Duo())
}
})
})
}
func TestDataVersioningAvoidsUnnecessaryTruncation(t *testing.T) {
expFull := CapnpEncode("(val = 9, duo = 8, ptr1 = (val = 77), ptr2 = (val = 55))", "VerTwoDataTwoPtr")
//expEmpty := CapnpEncode("()", "VerEmpty")
cv.Convey("Given a struct with 0 ptr fields, and a newer version of the struct with two data and two pointer fields", t, func() {
cv.Convey("then old code expecting the smaller struct but reading the newer-bigger struct should not truncate it if it doesn't have to (e.g. not assigning into a composite list), and should preserve all data when re-serializing it.", func() {
seg := capn.NewBuffer(nil)
scratch := capn.NewBuffer(nil)
big := air.NewRootVerTwoDataTwoPtr(seg)
one := air.NewVerOneData(scratch)
one.SetVal(77)
two := air.NewVerOneData(scratch)
two.SetVal(55)
big.SetVal(9)
big.SetDuo(8)
big.SetPtr1(one)
big.SetPtr2(two)
bigVerBytes := ShowSeg("\n\n with our 2x2 new big struct, segment seg is:", seg)
cv.So(bigVerBytes, cv.ShouldResemble, expFull)
// now pretend to be an old client, reading and writing
// expecting an empty struct, but full data should be preserved
// and written, because we aren't writing into a cramped/
// fixed-space composite-list space.
// Before test, verify that if we force reading into text-form, we get
// what we expect.
actEmptyCap := string(CapnpDecode(bigVerBytes, "VerEmpty"))
cv.So(actEmptyCap, cv.ShouldResemble, "()\n")
// okay, now the actual test:
weThinkEmptyButActuallyFull := air.ReadRootVerEmpty(seg)
freshSeg := capn.NewBuffer(nil)
wrapEmpty := air.NewRootWrapEmpty(freshSeg)
// here is the critical step, this should not truncate:
wrapEmpty.SetMightNotBeReallyEmpty(weThinkEmptyButActuallyFull)
// now verify:
freshBytes := ShowSeg("\n\n after wrapEmpty.SetMightNotBeReallyEmpty(weThinkEmptyButActuallyFull), segment freshSeg is:", freshSeg)
reseg, _, err := capn.ReadFromMemoryZeroCopy(freshBytes)
if err != nil {
panic(err)
}
ShowSeg(" after re-reading freshBytes, segment reseg is:", reseg)
fmt.Printf("freshBytes decoded by capnp as Wrap2x2: '%s'\n", string(CapnpDecode(freshBytes, "Wrap2x2")))
wrap22 := air.ReadRootWrap2x2plus(reseg)
notEmpty := wrap22.MightNotBeReallyEmpty()
val := notEmpty.Val()
cv.So(val, cv.ShouldEqual, 9)
duo := notEmpty.Duo()
cv.So(duo, cv.ShouldEqual, 8)
ptr1 := notEmpty.Ptr1()
ptr2 := notEmpty.Ptr2()
cv.So(ptr1.Val(), cv.ShouldEqual, 77)
cv.So(ptr2.Val(), cv.ShouldEqual, 55)
// Tre should get the default, as it was never set
cv.So(notEmpty.Tre(), cv.ShouldEqual, 0)
// same for Lst3
cv.So(notEmpty.Lst3().Len(), cv.ShouldEqual, 0)
})
})
}