Exemple #1
0
func Test002ListListStructList(t *testing.T) {

	cv.Convey("Given type RWTest struct { NestMatrix [][]Nester1; } in go, where Nester1 is a struct, and a mirror/parallel capnp struct air.RWTestCapn { nestMatrix @0: List(List(Nester1Capn)); } defined in the aircraftlib schema", t, func() {
		cv.Convey("When we Save() RWTest to capn and then Load() it back, the data should match, so that we have working List(List(Struct)) serialization and deserializatoin in go-capnproto", func() {

			// full RWTest
			rw := RWTest{
				NestMatrix: [][]Nester1{[]Nester1{Nester1{Strs: []string{"z", "w"}}, Nester1{Strs: []string{"q", "r"}}}, []Nester1{Nester1{Strs: []string{"zebra", "wally"}}, Nester1{Strs: []string{"qubert", "rocks"}}}},
			}

			var o bytes.Buffer
			rw.Save(&o)

			seg, n, err := capn.ReadFromMemoryZeroCopy(o.Bytes())
			cv.So(err, cv.ShouldEqual, nil)
			cv.So(n, cv.ShouldBeGreaterThan, 0)

			text := CapnpDecodeSegment(seg, "", "aircraftlib/aircraft.capnp", "RWTestCapn")

			if false {
				fmt.Printf("text = '%s'\n", text)
			}

			rw2 := &RWTest{}
			rw2.Load(&o)

			//fmt.Printf("rw = '%#v'\n", rw)
			//fmt.Printf("rw2 = '%#v'\n", rw2)

			same := reflect.DeepEqual(&rw, rw2)
			cv.So(same, cv.ShouldEqual, true)
		})
	})
}
Exemple #2
0
// start with smaller Struct(List)
func Test001StructList(t *testing.T) {

	cv.Convey("Given type Nester1 struct { Strs []string } in go, where Nester1 is a struct, and a mirror/parallel capnp struct air.Nester1Capn { strs @0: List(Text); } defined in the aircraftlib schema", t, func() {
		cv.Convey("When we Save() Nester to capn and then Load() it back, the data should match, so that we have working Struct(List) serialization and deserializatoin in go-capnproto", func() {

			// Does Nester1 alone serialization and deser okay?
			rw := Nester1{Strs: []string{"xenophilia", "watchowski"}}

			var o bytes.Buffer
			rw.Save(&o)

			seg, n, err := capn.ReadFromMemoryZeroCopy(o.Bytes())
			cv.So(err, cv.ShouldEqual, nil)
			cv.So(n, cv.ShouldBeGreaterThan, 0)

			text := CapnpDecodeSegment(seg, "", "aircraftlib/aircraft.capnp", "Nester1Capn")
			if false {
				fmt.Printf("text = '%s'\n", text)
			}
			rw2 := &Nester1{}
			rw2.Load(&o)

			//fmt.Printf("rw = '%#v'\n", rw)
			//fmt.Printf("rw2 = '%#v'\n", rw2)

			same := reflect.DeepEqual(&rw, rw2)
			cv.So(same, cv.ShouldEqual, true)
		})
	})
}
Exemple #3
0
// disk file of a capn segment -> in-memory capn segment -> stdin to capnp decode -> stdout human-readble string form
func CapnFileToText(serializedCapnpFilePathToDisplay string, capnpSchemaFilePath string, capnpExePath string) (string, error) {

	// a) read file into Segment

	byteslice, err := ioutil.ReadFile(serializedCapnpFilePathToDisplay)
	if err != nil {
		return "", err
	}

	seg, nbytes, err := capn.ReadFromMemoryZeroCopy(byteslice)

	if err == io.EOF {
		return "", err
	}
	if err != nil {
		return "", err
	}
	if nbytes == 0 {
		return "", errors.New(fmt.Sprintf("did not expect 0 bytes back from capn.ReadFromMemoryZeroCopy() on reading file '%s'", serializedCapnpFilePathToDisplay))
	}

	// b) tell CapnpDecodeSegment() to show the human-readable-text form of the message
	// warning: CapnpDecodeSegment() may panic on you. It is a testing utility so that
	//  is desirable. For production, do something else.
	return CapnpDecodeSegment(seg, capnpExePath, capnpSchemaFilePath, "Z"), nil
}
Exemple #4
0
func ProposerFromData(pm *ProposerManager, txnId *common.TxnId, data []byte) *Proposer {
	seg, _, err := capn.ReadFromMemoryZeroCopy(data)
	if err != nil {
		log.Println("Unable to decode proposer state", data)
	}
	// If we were on disk, then that means we must be locally complete
	// and just need to send out TLCs.
	state := msgs.ReadRootProposerState(seg)
	acceptorsCap := state.Acceptors()
	acceptors := make([]common.RMId, acceptorsCap.Len())
	for idx := range acceptors {
		acceptors[idx] = common.RMId(acceptorsCap.At(idx))
	}
	// We were on disk. Thus we received outcomes from all
	// acceptors. So we don't need to worry about the outcome
	// accumulator's fInc, hence just use -1 here.
	p := &Proposer{
		proposerManager: pm,
		mode:            proposerTLCSender,
		txnId:           txnId,
		acceptors:       acceptors,
		fInc:            -1,
	}
	p.init()
	p.allAcceptorsAgreed = true
	return p
}
Exemple #5
0
func VarFromData(data []byte, exe *dispatcher.Executor, disk *mdbs.MDBServer, vm *VarManager) (*Var, error) {
	seg, _, err := capn.ReadFromMemoryZeroCopy(data)
	if err != nil {
		return nil, err
	}
	varCap := msgs.ReadRootVar(seg)

	v := newVar(common.MakeVarUUId(varCap.Id()), exe, disk, vm)
	positions := varCap.Positions()
	if positions.Len() != 0 {
		v.positions = (*common.Positions)(&positions)
	}

	writeTxnId := common.MakeTxnId(varCap.WriteTxnId())
	writeTxnClock := VectorClockFromCap(varCap.WriteTxnClock())
	writesClock := VectorClockFromCap(varCap.WritesClock())
	server.Log(v.UUId, "Restored", writeTxnId)

	if result, err := disk.ReadonlyTransaction(func(rtxn *mdbs.RTxn) (interface{}, error) {
		return db.ReadTxnFromDisk(rtxn, writeTxnId)
	}).ResultError(); err == nil {
		if result == nil || result.(*msgs.Txn) == nil {
			panic(fmt.Sprintf("%v Unable to find txn %v on disk (%v)", v.UUId, writeTxnId, result))
		}
		actions := result.(*msgs.Txn).Actions()
		v.curFrame = NewFrame(nil, v, writeTxnId, &actions, writeTxnClock, writesClock)
		v.curFrameOnDisk = v.curFrame
	} else {
		return nil, err
	}

	v.varCap = &varCap

	return v, nil
}
Exemple #6
0
func (cah *connectionAwaitHandshake) readAndDecryptOne() (*capn.Segment, error) {
	if cah.sessionKey == nil {
		return capn.ReadFromStream(cah.socket, nil)
	}
	read, err := cah.socket.Read(cah.inBuff)
	if err != nil {
		return nil, err
	} else if read < len(cah.inBuff) {
		return nil, fmt.Errorf("Only read %v bytes, wanted %v", read, len(cah.inBuff))
	}
	copy(cah.nonceAryIn[16:], cah.inBuff[:8])
	msgLen := binary.BigEndian.Uint64(cah.inBuff[8:16])
	plainLen := msgLen - secretbox.Overhead
	msgBuf := make([]byte, plainLen+msgLen)
	for recvBuf := msgBuf[plainLen:]; len(recvBuf) != 0; {
		read, err = cah.socket.Read(recvBuf)
		if err != nil {
			return nil, err
		} else {
			recvBuf = recvBuf[read:]
		}
	}
	plaintext, ok := secretbox.Open(msgBuf[:0], msgBuf[plainLen:], cah.nonceAryIn, cah.sessionKey)
	if !ok {
		return nil, fmt.Errorf("Unable to decrypt message")
	}
	seg, _, err := capn.ReadFromMemoryZeroCopy(plaintext)
	return seg, err
}
Exemple #7
0
func TopologyDeserialize(txnId *common.TxnId, root *msgs.VarIdPos, data []byte) (*Topology, error) {
	seg, _, err := capn.ReadFromMemoryZeroCopy(data)
	if err != nil {
		return nil, err
	}
	topology := msgs.ReadRootTopology(seg)
	return TopologyFromCap(txnId, root, &topology), nil
}
Exemple #8
0
func TopologyFromCap(txnId *common.TxnId, root *msgs.VarIdPos, data []byte) (*Topology, error) {
	seg, _, err := capn.ReadFromMemoryZeroCopy(data)
	if err != nil {
		return nil, err
	}
	configCap := msgs.ReadRootConfiguration(seg)
	config := ConfigurationFromCap(&configCap)
	return NewTopology(txnId, root, config), nil
}
Exemple #9
0
func TestDataVersioningZeroPointersToMore(t *testing.T) {

	expEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerEmptyList")

	cv.Convey("Given a struct with 0 ptr fields, and a newer version of the struct with 1-2 pointer fields", t, func() {
		cv.Convey("then serializing the empty list and reading it back into 1 or 2 pointer fields should default initialize the pointer fields", func() {

			seg := capn.NewBuffer(nil)
			scratch := capn.NewBuffer(nil)

			emptyholder := air.NewRootHoldsVerEmptyList(seg)
			elist := air.NewVerEmptyList(scratch, 2)
			emptyholder.SetMylist(elist)

			actEmpty := ShowSeg("          after NewRootHoldsVerEmptyList(seg) and SetMylist(elist), segment seg is:", seg)
			actEmptyCap := string(CapnpDecode(actEmpty, "HoldsVerEmptyList"))
			expEmptyCap := string(CapnpDecode(expEmpty, "HoldsVerEmptyList"))
			cv.So(actEmptyCap, cv.ShouldResemble, expEmptyCap)

			fmt.Printf("\n actEmpty is \n")
			ShowBytes(actEmpty, 10)
			fmt.Printf("actEmpty decoded by capnp: '%s'\n", string(CapnpDecode(actEmpty, "HoldsVerEmptyList")))
			cv.So(actEmpty, cv.ShouldResemble, expEmpty)

			// seg is set, now read into bigger list
			buf := bytes.Buffer{}
			seg.WriteTo(&buf)
			segbytes := buf.Bytes()

			reseg, _, err := capn.ReadFromMemoryZeroCopy(segbytes)
			if err != nil {
				panic(err)
			}
			ShowSeg("      after re-reading segbytes, segment reseg is:", reseg)
			fmt.Printf("segbytes decoded by capnp as HoldsVerOneDataList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerOneDataList")))

			reHolder := air.ReadRootHoldsVerTwoTwoList(reseg)
			list22 := reHolder.Mylist()
			len22 := list22.Len()
			cv.So(len22, cv.ShouldEqual, 2)
			for i := 0; i < 2; i++ {
				ele := list22.At(i)
				val := ele.Val()
				cv.So(val, cv.ShouldEqual, 0)
				duo := ele.Duo()
				cv.So(duo, cv.ShouldEqual, 0)
				ptr1 := ele.Ptr1()
				ptr2 := ele.Ptr2()
				fmt.Printf("ptr1 = %#v\n", ptr1)
				cv.So(ptr1.Segment, cv.ShouldEqual, nil)
				fmt.Printf("ptr2 = %#v\n", ptr2)
				cv.So(ptr2.Segment, cv.ShouldEqual, nil)
			}

		})
	})
}
Exemple #10
0
func VarFromData(data []byte, exe *dispatcher.Executor, db *db.Databases, vm *VarManager) (*Var, error) {
	seg, _, err := capn.ReadFromMemoryZeroCopy(data)
	if err != nil {
		return nil, err
	}
	varCap := msgs.ReadRootVar(seg)

	v := newVar(common.MakeVarUUId(varCap.Id()), exe, db, vm)
	positions := varCap.Positions()
	if positions.Len() != 0 {
		v.positions = (*common.Positions)(&positions)
	}

	writeTxnId := common.MakeTxnId(varCap.WriteTxnId())
	writeTxnClock := VectorClockFromCap(varCap.WriteTxnClock())
	writesClock := VectorClockFromCap(varCap.WritesClock())
	server.Log(v.UUId, "Restored", writeTxnId)

	if result, err := db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
		return db.ReadTxnBytesFromDisk(rtxn, writeTxnId)
	}).ResultError(); err == nil && result != nil {
		bites := result.([]byte)
		if seg, _, err := capn.ReadFromMemoryZeroCopy(bites); err == nil {
			txn := msgs.ReadRootTxn(seg)
			actions := txn.Actions()
			v.curFrame = NewFrame(nil, v, writeTxnId, &actions, writeTxnClock, writesClock)
			v.curFrameOnDisk = v.curFrame
			v.varCap = &varCap
			return v, nil
		} else {
			return nil, err
		}
	} else {
		return nil, err
	}
}
Exemple #11
0
func loadVars(disk *mdbs.MDBServer, vars map[common.VarUUId]*varstate) {
	_, err := disk.ReadonlyTransaction(func(rtxn *mdbs.RTxn) (interface{}, error) {
		return rtxn.WithCursor(db.DB.Vars, func(cursor *mdb.Cursor) (interface{}, error) {
			key, data, err := cursor.Get(nil, nil, mdb.FIRST)
			for ; err == nil; key, data, err = cursor.Get(nil, nil, mdb.NEXT) {
				vUUId := common.MakeVarUUId(key)
				seg, _, err := capn.ReadFromMemoryZeroCopy(data)
				if err != nil {
					log.Println(err)
					continue
				}

				varCap := msgs.ReadRootVar(seg)
				pos := varCap.Positions()
				positions := (*common.Positions)(&pos)
				writeTxnId := common.MakeTxnId(varCap.WriteTxnId())
				writeTxnClock := eng.VectorClockFromCap(varCap.WriteTxnClock())
				writesClock := eng.VectorClockFromCap(varCap.WritesClock())

				if state, found := vars[*vUUId]; found {
					if err := state.matches(disk, writeTxnId, writeTxnClock, writesClock, positions); err != nil {
						log.Println(err)
					}
				} else {
					state = &varstate{
						vUUId:            vUUId,
						disks:            []*mdbs.MDBServer{disk},
						writeTxnId:       writeTxnId,
						writeTxnClock:    writeTxnClock,
						writeWritesClock: writesClock,
						positions:        positions,
					}
					vars[*vUUId] = state
				}
			}
			if err == mdb.NotFound {
				return nil, nil
			} else {
				return nil, err
			}
		})
	}).ResultError()
	if err != nil {
		log.Println(err)
	}
}
Exemple #12
0
func (vw *varWrapper) start() {
	defer close(vw.c)

	c1 := &varWrapperCell{varWrapper: vw}
	c2 := &varWrapperCell{varWrapper: vw}
	c1.other, c2.other = c2, c1

	curCell := c1
	_, err := vw.store.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
		rtxn.WithCursor(vw.store.db.Vars, func(cursor *mdbs.Cursor) interface{} {
			vUUIdBytes, varBytes, err := cursor.Get(nil, nil, mdb.FIRST)
			if err != nil {
				cursor.Error(fmt.Errorf("Err on finding first var in %v: %v", vw.store, err))
				return nil
			}
			if !bytes.Equal(vUUIdBytes, configuration.TopologyVarUUId[:]) {
				vUUId := common.MakeVarUUId(vUUIdBytes)
				cursor.Error(fmt.Errorf("Err on finding first var in %v: expected to find topology var, but found %v instead! (%v)", vw.store, vUUId, varBytes))
				return nil
			}
			for ; err == nil; vUUIdBytes, varBytes, err = cursor.Get(nil, nil, mdb.NEXT) {
				vUUId := common.MakeVarUUId(vUUIdBytes)
				seg, _, err := capn.ReadFromMemoryZeroCopy(varBytes)
				if err != nil {
					cursor.Error(fmt.Errorf("Err on decoding %v in %v: %v (%v)", vUUId, vw.store, err, varBytes))
					return nil
				}
				varCap := msgs.ReadRootVar(seg)
				curCell.vUUId = vUUId
				curCell.varCap = &varCap
				vw.c <- curCell
				curCell = curCell.other
			}
			if err != nil && err != mdb.NotFound {
				cursor.Error(err)
			}
			return nil
		})
		return nil
	}).ResultError()
	if err != nil {
		curCell.err = err
		vw.c <- curCell
	}
}
Exemple #13
0
func ReadTxnFromDisk(rtxn *mdbs.RTxn, txnId *common.TxnId) (*msgs.Txn, error) {
	bites, err := rtxn.Get(DB.Transactions, txnId[:])
	switch err {
	case nil:
		if seg, _, err := capn.ReadFromMemoryZeroCopy(bites); err == nil {
			txn := msgs.ReadRootTxn(seg)
			return &txn, nil
		} else {
			return nil, err
		}

	case mdb.NotFound:
		return nil, nil

	default:
		return nil, err
	}
}
Exemple #14
0
func (am *AcceptorManager) loadFromData(txnId *common.TxnId, data []byte) {
	seg, _, err := capn.ReadFromMemoryZeroCopy(data)
	if err != nil {
		log.Println("Unable to decode acceptor state", data)
		return
	}
	state := msgs.ReadRootAcceptorState(seg)
	txn := state.Txn()

	instId := instanceId([instanceIdLen]byte{})
	instIdSlice := instId[:]

	outcome := state.Outcome()
	copy(instIdSlice, txnId[:])

	instances := state.Instances()
	acc := AcceptorFromData(txnId, &txn, &outcome, state.SendToAll(), &instances, am)
	aInst := &acceptorInstances{acceptor: acc}
	am.acceptors[*txnId] = aInst

	for idx, l := 0, instances.Len(); idx < l; idx++ {
		instancesForVar := instances.At(idx)
		vUUId := common.MakeVarUUId(instancesForVar.VarId())
		acceptedInstances := instancesForVar.Instances()
		for idy, m := 0, acceptedInstances.Len(); idy < m; idy++ {
			acceptedInstance := acceptedInstances.At(idy)
			roundNumber := acceptedInstance.RoundNumber()
			ballot := acceptedInstance.Ballot()
			instance := &instance{
				manager:     am,
				vUUId:       vUUId,
				promiseNum:  paxosNumber(roundNumber),
				acceptedNum: paxosNumber(roundNumber),
				accepted:    &ballot,
			}
			binary.BigEndian.PutUint32(instIdSlice[common.KeyLen:], acceptedInstance.RmId())
			copy(instIdSlice[common.KeyLen+4:], vUUId[:])
			am.instances[instId] = instance
			aInst.addInstance(&instId)
		}
	}

	acc.Start()
}
Exemple #15
0
func TestV1DataVersioningEmptyToBigger(t *testing.T) {

	//expOneSet := CapnpEncode("(mylist = [(val = 27),(val = 42)])", "HoldsVerOneDataList")
	//expOneZeroed := CapnpEncode("(mylist = [(val = 0),(val = 0)])", "HoldsVerOneDataList")
	//expOneEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerOneDataList")
	expEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerEmptyList")

	cv.Convey("Given a struct with 0 data/0 ptr fields, and a newer version of the struct with 1 data fields", t, func() {
		cv.Convey("then reading from serialized form the small list into the bigger (one or two data values) list should work, getting default value 0 for val/duo.", func() {

			seg := capn.NewBuffer(nil)
			scratch := capn.NewBuffer(nil)

			emptyholder := air.NewRootHoldsVerEmptyList(seg)
			elist := air.NewVerEmptyList(scratch, 2)
			emptyholder.SetMylist(elist)

			actEmpty := ShowSeg("          after NewRootHoldsVerEmptyList(seg) and SetMylist(elist), segment seg is:", seg)
			actEmptyCap := string(CapnpDecode(actEmpty, "HoldsVerEmptyList"))
			expEmptyCap := string(CapnpDecode(expEmpty, "HoldsVerEmptyList"))
			cv.So(actEmptyCap, cv.ShouldResemble, expEmptyCap)

			fmt.Printf("\n actEmpty is \n")
			ShowBytes(actEmpty, 10)
			fmt.Printf("actEmpty decoded by capnp: '%s'\n", string(CapnpDecode(actEmpty, "HoldsVerEmptyList")))
			cv.So(actEmpty, cv.ShouldResemble, expEmpty)

			// seg is set, now read into bigger list
			buf := bytes.Buffer{}
			seg.WriteTo(&buf)
			segbytes := buf.Bytes()

			reseg, _, err := capn.ReadFromMemoryZeroCopy(segbytes)
			if err != nil {
				panic(err)
			}
			ShowSeg("      after re-reading segbytes, segment reseg is:", reseg)
			fmt.Printf("segbytes decoded by capnp as HoldsVerOneDataList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerOneDataList")))

			reHolder := air.ReadRootHoldsVerOneDataList(reseg)
			onelist := reHolder.Mylist()
			lenone := onelist.Len()
			cv.So(lenone, cv.ShouldEqual, 2)
			for i := 0; i < 2; i++ {
				ele := onelist.At(i)
				val := ele.Val()
				cv.So(val, cv.ShouldEqual, 0)
			}

			reHolder2 := air.ReadRootHoldsVerTwoDataList(reseg)
			twolist := reHolder2.Mylist()
			lentwo := twolist.Len()
			cv.So(lentwo, cv.ShouldEqual, 2)
			for i := 0; i < 2; i++ {
				ele := twolist.At(i)
				val := ele.Val()
				cv.So(val, cv.ShouldEqual, 0)
				duo := ele.Duo()
				cv.So(duo, cv.ShouldEqual, 0)
			}

		})
	})
}
Exemple #16
0
func TestDataVersioningZeroPointersToTwo(t *testing.T) {
	cv.Convey("Given a struct with 2 ptr fields, and another version of the struct with 0 or 1 pointer fields", t, func() {
		cv.Convey("then reading serialized bigger-struct-list into the smaller (empty or one data-pointer) list should work, truncating/ignoring the new fields", func() {

			seg := capn.NewBuffer(nil)
			scratch := capn.NewBuffer(nil)
			holder := air.NewRootHoldsVerTwoTwoList(seg)

			twolist := air.NewVerTwoDataTwoPtrList(scratch, 2)
			plist := capn.PointerList(twolist)

			d0 := air.NewVerTwoDataTwoPtr(scratch)
			d0.SetVal(27)
			d0.SetDuo(26)

			v1 := air.NewVerOneData(scratch)
			v1.SetVal(25)
			v2 := air.NewVerOneData(scratch)
			v2.SetVal(23)

			d0.SetPtr1(v1)
			d0.SetPtr2(v2)

			d1 := air.NewVerTwoDataTwoPtr(scratch)
			d1.SetVal(42)
			d1.SetDuo(41)

			w1 := air.NewVerOneData(scratch)
			w1.SetVal(40)
			w2 := air.NewVerOneData(scratch)
			w2.SetVal(38)

			d1.SetPtr1(w1)
			d1.SetPtr2(w2)

			plist.Set(0, capn.Object(d0))
			plist.Set(1, capn.Object(d1))

			holder.SetMylist(twolist)

			ShowSeg("     before serializing out, segment scratch is:", scratch)
			ShowSeg("     before serializing out, segment seg is:", seg)

			// serialize out
			buf := bytes.Buffer{}
			seg.WriteTo(&buf)
			segbytes := buf.Bytes()

			// and read-back in using smaller expectations
			reseg, _, err := capn.ReadFromMemoryZeroCopy(segbytes)
			if err != nil {
				panic(err)
			}
			ShowSeg("      after re-reading segbytes, segment reseg is:", reseg)
			fmt.Printf("segbytes decoded by capnp as HoldsVerEmptyList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerEmptyList")))
			fmt.Printf("segbytes decoded by capnp as HoldsVerOnePtrList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerOnePtrList")))
			fmt.Printf("segbytes decoded by capnp as HoldsVerTwoTwoList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerTwoTwoList")))

			reHolder := air.ReadRootHoldsVerEmptyList(reseg)
			elist := reHolder.Mylist()
			lene := elist.Len()
			cv.So(lene, cv.ShouldEqual, 2)

			reHolder1 := air.ReadRootHoldsVerOnePtrList(reseg)
			onelist := reHolder1.Mylist()
			lenone := onelist.Len()
			cv.So(lenone, cv.ShouldEqual, 2)

			for i := 0; i < 2; i++ {
				ele := onelist.At(i)
				ptr1 := ele.Ptr()
				cv.So(ptr1.Val(), cv.ShouldEqual, twolist.At(i).Ptr1().Val())
			}

			reHolder2 := air.ReadRootHoldsVerTwoTwoPlus(reseg)
			twolist2 := reHolder2.Mylist()
			lentwo2 := twolist2.Len()
			cv.So(lentwo2, cv.ShouldEqual, 2)

			for i := 0; i < 2; i++ {
				ele := twolist2.At(i)
				ptr1 := ele.Ptr1()
				ptr2 := ele.Ptr2()
				cv.So(ptr1.Val(), cv.ShouldEqual, twolist.At(i).Ptr1().Val())
				//cv.So(ptr1.Duo(), cv.ShouldEqual, twolist.At(i).Ptr1().Duo())
				cv.So(ptr2.Val(), cv.ShouldEqual, twolist.At(i).Ptr2().Val())
				//cv.So(ptr2.Duo(), cv.ShouldEqual, twolist.At(i).Ptr2().Duo())
				cv.So(ele.Tre(), cv.ShouldEqual, 0)
				cv.So(ele.Lst3().Len(), cv.ShouldEqual, 0)
			}

		})
	})
}
Exemple #17
0
func (lc *locationChecker) locationCheck(cell *varWrapperCell) error {
	vUUId := cell.vUUId
	varCap := cell.varCap
	foundIn := cell.store
	fmt.Printf("%v %v\n", foundIn, vUUId)
	txnId := common.MakeTxnId(varCap.WriteTxnId())

	res, err := foundIn.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
		return foundIn.db.ReadTxnBytesFromDisk(rtxn, txnId)
	}).ResultError()
	if err != nil {
		return err
	}
	txnBites, ok := res.([]byte)
	if res == nil || (ok && txnBites == nil) {
		return fmt.Errorf("Failed to find %v from %v in %v", txnId, vUUId, foundIn)
	}
	seg, _, err := capn.ReadFromMemoryZeroCopy(txnBites)
	if err != nil {
		return err
	}
	txnCap := msgs.ReadRootTxn(seg)

	positions := varCap.Positions().ToArray()
	rmIds, err := lc.resolver.ResolveHashCodes(positions)
	if err != nil {
		return err
	}
	foundLocal := false
	for _, rmId := range rmIds {
		if foundLocal = rmId == foundIn.rmId; foundLocal {
			break
		}
	}
	if !foundLocal {
		// It must have emigrated but we don't delete.
		txnId = nil
	}
	for _, rmId := range rmIds {
		if rmId == foundIn.rmId {
			continue
		} else if remote, found := lc.stores[rmId]; found {
			res, err := remote.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
				bites, err := rtxn.Get(remote.db.Vars, vUUId[:])
				if err == mdb.NotFound {
					return nil
				} else if err == nil {
					return bites
				} else {
					return nil
				}
			}).ResultError()
			if err != nil {
				return err
			}
			varBites, ok := res.([]byte)
			if res == nil || (ok && varBites == nil) {
				if vUUId.BootCount() == 1 && vUUId.ConnectionCount() == 0 &&
					(txnId == nil ||
						(txnId.BootCount() == 1 && txnId.ConnectionCount() == 0 &&
							txnCap.Actions().Len() == 1 && txnCap.Actions().At(0).Which() == msgs.ACTION_CREATE)) {
					fmt.Printf("Failed to find %v in %v (%v, %v, %v) but it looks like it's a bad root.\n", vUUId, remote, rmIds, positions, foundIn)
				} else {
					return fmt.Errorf("Failed to find %v in %v (%v, %v, %v)", vUUId, remote, rmIds, positions, foundIn)
				}
			} else {
				seg, _, err := capn.ReadFromMemoryZeroCopy(varBites)
				if err != nil {
					return err
				}
				remoteTxnId := common.MakeTxnId(msgs.ReadRootVar(seg).WriteTxnId())
				if txnId == nil {
					txnId = remoteTxnId
				}
				if remoteTxnId.Compare(txnId) != common.EQ {
					return fmt.Errorf("%v on %v is at %v; on %v is at %v", vUUId, foundIn, txnId, remote, remoteTxnId)
				}
			}
		}
	}
	return nil
}
Exemple #18
0
func (s *store) LoadTopology() error {
	res, err := s.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
		bites, err := rtxn.Get(s.db.Vars, configuration.TopologyVarUUId[:])
		if err != nil {
			rtxn.Error(err)
			return nil
		}
		seg, _, err := capn.ReadFromMemoryZeroCopy(bites)
		if err != nil {
			rtxn.Error(err)
			return nil
		}
		varCap := msgs.ReadRootVar(seg)
		txnId := common.MakeTxnId(varCap.WriteTxnId())
		bites = s.db.ReadTxnBytesFromDisk(rtxn, txnId)
		if bites == nil {
			rtxn.Error(fmt.Errorf("Unable to find txn for topology: %v", txnId))
			return nil
		}
		seg, _, err = capn.ReadFromMemoryZeroCopy(bites)
		if err != nil {
			rtxn.Error(err)
			return nil
		}
		txnCap := msgs.ReadRootTxn(seg)
		actions := txnCap.Actions()
		if actions.Len() != 1 {
			rtxn.Error(fmt.Errorf("Topology txn has %v actions; expected 1", actions.Len()))
			return nil
		}
		action := actions.At(0)
		var refs msgs.VarIdPos_List
		switch action.Which() {
		case msgs.ACTION_WRITE:
			w := action.Write()
			bites = w.Value()
			refs = w.References()
		case msgs.ACTION_READWRITE:
			rw := action.Readwrite()
			bites = rw.Value()
			refs = rw.References()
		case msgs.ACTION_CREATE:
			c := action.Create()
			bites = c.Value()
			refs = c.References()
		default:
			rtxn.Error(fmt.Errorf("Expected topology txn action to be w, rw, or c; found %v", action.Which()))
			return nil
		}

		if refs.Len() != 1 {
			rtxn.Error(fmt.Errorf("Topology txn action has %v references; expected 1", refs.Len()))
			return nil
		}
		rootRef := refs.At(0)

		seg, _, err = capn.ReadFromMemoryZeroCopy(bites)
		if err != nil {
			rtxn.Error(err)
			return nil
		}
		topology, err := configuration.TopologyFromCap(txnId, &rootRef, bites)
		if err != nil {
			rtxn.Error(err)
			return nil
		}
		return topology
	}).ResultError()
	if err != nil {
		return err
	}
	s.topology = res.(*configuration.Topology)
	return nil
}
Exemple #19
0
// paxos.Connection interface to allow sending to ourself.
func (cm *ConnectionManager) Send(b []byte) {
	seg, _, err := capn.ReadFromMemoryZeroCopy(b)
	server.CheckFatal(err)
	msg := msgs.ReadRootMessage(seg)
	cm.DispatchMessage(cm.RMId, msg.Which(), &msg)
}
Exemple #20
0
func TestV1DataVersioningBiggerToEmpty(t *testing.T) {

	//expTwoSet := CapnpEncode("(mylist = [(val = 27, duo = 26),(val = 42, duo = 41)])", "HoldsVerTwoDataList")
	//expOneDataOneDefault := CapnpEncode("(mylist = [(val = 27, duo = 0),(val = 42, duo = 0)])", "HoldsVerTwoDataList")
	//expTwoEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerTwoDataList")

	//expEmpty := CapnpEncode("(mylist = [(),()])", "HoldsVerEmptyList")
	//expOne := CapnpEncode("(mylist = [(val = 27),(val = 42)])", "HoldsVerOneDataList")

	cv.Convey("Given a struct with 0 data/0 ptr fields, and a newer version of the struct with 2 data fields", t, func() {
		cv.Convey("then reading serialized bigger-struct-list into the smaller (empty or one data-member) list should work, truncating/ignoring the new fields", func() {

			seg := capn.NewBuffer(nil)
			scratch := capn.NewBuffer(nil)
			holder := air.NewRootHoldsVerTwoDataList(seg)

			twolist := air.NewVerTwoDataList(scratch, 2)
			plist := capn.PointerList(twolist)

			d0 := air.NewVerTwoData(scratch)
			d0.SetVal(27)
			d0.SetDuo(26)
			d1 := air.NewVerTwoData(scratch)
			d1.SetVal(42)
			d1.SetDuo(41)
			plist.Set(0, capn.Object(d0))
			plist.Set(1, capn.Object(d1))

			holder.SetMylist(twolist)

			ShowSeg("     before serializing out, segment scratch is:", scratch)
			ShowSeg("     before serializing out, segment seg is:", seg)

			// serialize out
			buf := bytes.Buffer{}
			seg.WriteTo(&buf)
			segbytes := buf.Bytes()

			// and read-back in using smaller expectations
			reseg, _, err := capn.ReadFromMemoryZeroCopy(segbytes)
			if err != nil {
				panic(err)
			}
			ShowSeg("      after re-reading segbytes, segment reseg is:", reseg)
			fmt.Printf("segbytes decoded by capnp as HoldsVerEmptyList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerEmptyList")))
			fmt.Printf("segbytes decoded by capnp as HoldsVerOneDataList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerOneDataList")))
			fmt.Printf("segbytes decoded by capnp as HoldsVerTwoDataList: '%s'\n", string(CapnpDecode(segbytes, "HoldsVerTwoDataList")))

			reHolder := air.ReadRootHoldsVerEmptyList(reseg)
			elist := reHolder.Mylist()
			lene := elist.Len()
			cv.So(lene, cv.ShouldEqual, 2)

			reHolder1 := air.ReadRootHoldsVerOneDataList(reseg)
			onelist := reHolder1.Mylist()
			lenone := onelist.Len()
			cv.So(lenone, cv.ShouldEqual, 2)

			for i := 0; i < 2; i++ {
				ele := onelist.At(i)
				val := ele.Val()
				cv.So(val, cv.ShouldEqual, twolist.At(i).Val())
			}

			reHolder2 := air.ReadRootHoldsVerTwoDataList(reseg)
			twolist2 := reHolder2.Mylist()
			lentwo2 := twolist2.Len()
			cv.So(lentwo2, cv.ShouldEqual, 2)

			for i := 0; i < 2; i++ {
				ele := twolist2.At(i)
				val := ele.Val()
				duo := ele.Duo()
				cv.So(val, cv.ShouldEqual, twolist.At(i).Val())
				cv.So(duo, cv.ShouldEqual, twolist.At(i).Duo())
			}

		})
	})
}
Exemple #21
0
func TestDataVersioningAvoidsUnnecessaryTruncation(t *testing.T) {

	expFull := CapnpEncode("(val = 9, duo = 8, ptr1 = (val = 77), ptr2 = (val = 55))", "VerTwoDataTwoPtr")
	//expEmpty := CapnpEncode("()", "VerEmpty")

	cv.Convey("Given a struct with 0 ptr fields, and a newer version of the struct with two data and two pointer fields", t, func() {
		cv.Convey("then old code expecting the smaller struct but reading the newer-bigger struct should not truncate it if it doesn't have to (e.g. not assigning into a composite list), and should preserve all data when re-serializing it.", func() {

			seg := capn.NewBuffer(nil)
			scratch := capn.NewBuffer(nil)

			big := air.NewRootVerTwoDataTwoPtr(seg)
			one := air.NewVerOneData(scratch)
			one.SetVal(77)
			two := air.NewVerOneData(scratch)
			two.SetVal(55)
			big.SetVal(9)
			big.SetDuo(8)
			big.SetPtr1(one)
			big.SetPtr2(two)

			bigVerBytes := ShowSeg("\n\n   with our 2x2 new big struct, segment seg is:", seg)

			cv.So(bigVerBytes, cv.ShouldResemble, expFull)

			// now pretend to be an old client, reading and writing
			// expecting an empty struct, but full data should be preserved
			// and written, because we aren't writing into a cramped/
			// fixed-space composite-list space.

			// Before test, verify that if we force reading into text-form, we get
			// what we expect.
			actEmptyCap := string(CapnpDecode(bigVerBytes, "VerEmpty"))
			cv.So(actEmptyCap, cv.ShouldResemble, "()\n")

			// okay, now the actual test:
			weThinkEmptyButActuallyFull := air.ReadRootVerEmpty(seg)

			freshSeg := capn.NewBuffer(nil)
			wrapEmpty := air.NewRootWrapEmpty(freshSeg)

			// here is the critical step, this should not truncate:
			wrapEmpty.SetMightNotBeReallyEmpty(weThinkEmptyButActuallyFull)

			// now verify:
			freshBytes := ShowSeg("\n\n          after wrapEmpty.SetMightNotBeReallyEmpty(weThinkEmptyButActuallyFull), segment freshSeg is:", freshSeg)

			reseg, _, err := capn.ReadFromMemoryZeroCopy(freshBytes)
			if err != nil {
				panic(err)
			}
			ShowSeg("      after re-reading freshBytes, segment reseg is:", reseg)
			fmt.Printf("freshBytes decoded by capnp as Wrap2x2: '%s'\n", string(CapnpDecode(freshBytes, "Wrap2x2")))

			wrap22 := air.ReadRootWrap2x2plus(reseg)
			notEmpty := wrap22.MightNotBeReallyEmpty()
			val := notEmpty.Val()
			cv.So(val, cv.ShouldEqual, 9)
			duo := notEmpty.Duo()
			cv.So(duo, cv.ShouldEqual, 8)
			ptr1 := notEmpty.Ptr1()
			ptr2 := notEmpty.Ptr2()
			cv.So(ptr1.Val(), cv.ShouldEqual, 77)
			cv.So(ptr2.Val(), cv.ShouldEqual, 55)
			// Tre should get the default, as it was never set
			cv.So(notEmpty.Tre(), cv.ShouldEqual, 0)
			// same for Lst3
			cv.So(notEmpty.Lst3().Len(), cv.ShouldEqual, 0)
		})
	})
}