func (cr *connectionRun) handleTxnOutcome(outcome *msgs.ClientTxnOutcome) error {
txnId := common.MakeTxnId(outcome.Id())
if cr.liveTxn == nil {
return fmt.Errorf("Received txn outcome for unknown txn: %v", txnId)
}
finalTxnId := common.MakeTxnId(outcome.FinalId())
if !bytes.Equal(cr.liveTxn.txn.Id(), outcome.Id()) {
return fmt.Errorf("Received txn outcome for wrong txn: %v (expecting %v) (final %v) (which %v)", txnId, common.MakeTxnId(cr.liveTxn.txn.Id()), finalTxnId, outcome.Which())
}
var err error
final := binary.BigEndian.Uint64(finalTxnId[:8])
if final < cr.nextTxnId {
return fmt.Errorf("Final (%v) < next (%v)\n", final, cr.nextTxnId)
}
cr.nextTxnId = final + 1 + uint64(cr.rng.Intn(8))
var modifiedVars []*common.VarUUId
switch outcome.Which() {
case msgs.CLIENTTXNOUTCOME_COMMIT:
cr.cache.updateFromTxnCommit(cr.liveTxn.txn, finalTxnId)
case msgs.CLIENTTXNOUTCOME_ABORT:
updates := outcome.Abort()
modifiedVars = cr.cache.updateFromTxnAbort(&updates)
case msgs.CLIENTTXNOUTCOME_ERROR:
err = fmt.Errorf(outcome.Error())
}
if !cr.liveTxn.setOutcomeError(outcome, modifiedVars, err) {
return fmt.Errorf("Live txn already closed")
}
cr.liveTxn = nil
return nil
}
func (br badReads) combine(rmBal *rmBallot) {
clock := rmBal.ballot.Clock
badRead := rmBal.ballot.VoteCap.AbortBadRead()
txnId := common.MakeTxnId(badRead.TxnId())
actions := badRead.TxnActions()
for idx, l := 0, actions.Len(); idx < l; idx++ {
action := actions.At(idx)
vUUId := common.MakeVarUUId(action.VarId())
if bra, found := br[*vUUId]; found {
bra.combine(&action, rmBal, txnId, clock.Clock[*vUUId])
} else if action.Which() == msgs.ACTION_READ {
br[*vUUId] = &badReadAction{
rmBallot: rmBal,
vUUId: vUUId,
txnId: common.MakeTxnId(action.Read().Version()),
clockElem: clock.Clock[*vUUId] - 1,
action: &action,
}
if clock.Clock[*vUUId] == 0 {
panic(fmt.Sprintf("Just did 0 - 1 in int64 (%v, %v) (%v)", vUUId, clock, txnId))
}
} else {
br[*vUUId] = &badReadAction{
rmBallot: rmBal,
vUUId: vUUId,
txnId: txnId,
clockElem: clock.Clock[*vUUId],
action: &action,
}
}
}
}
func (pd *ProposerDispatcher) TwoBTxnVotesReceived(sender common.RMId, twoBTxnVotes *msgs.TwoBTxnVotes) {
var txnId *common.TxnId
switch twoBTxnVotes.Which() {
case msgs.TWOBTXNVOTES_FAILURES:
txnId = common.MakeTxnId(twoBTxnVotes.Failures().TxnId())
case msgs.TWOBTXNVOTES_OUTCOME:
txnId = common.MakeTxnId(twoBTxnVotes.Outcome().Txn().Id())
default:
panic(fmt.Sprintf("Unexpected 2BVotes type: %v", twoBTxnVotes.Which()))
}
pd.withProposerManager(txnId, func(pm *ProposerManager) { pm.TwoBTxnVotesReceived(sender, txnId, twoBTxnVotes) })
}
func (cash *connectionAwaitServerHandshake) verifyTopology(topology *server.Topology, remote *msgs.HelloFromServer) (bool, *server.Topology) {
remoteTopologyDBVersion := common.MakeTxnId(remote.TopologyDBVersion())
remoteTopologyCap := remote.Topology()
remoteRoot := remote.Root()
remoteTopology := server.TopologyFromCap(remoteTopologyDBVersion, &remoteRoot, &remoteTopologyCap)
return topology.Configuration.Equal(remoteTopology.Configuration), remoteTopology
}
func VarFromData(data []byte, exe *dispatcher.Executor, disk *mdbs.MDBServer, vm *VarManager) (*Var, error) {
seg, _, err := capn.ReadFromMemoryZeroCopy(data)
if err != nil {
return nil, err
}
varCap := msgs.ReadRootVar(seg)
v := newVar(common.MakeVarUUId(varCap.Id()), exe, disk, vm)
positions := varCap.Positions()
if positions.Len() != 0 {
v.positions = (*common.Positions)(&positions)
}
writeTxnId := common.MakeTxnId(varCap.WriteTxnId())
writeTxnClock := VectorClockFromCap(varCap.WriteTxnClock())
writesClock := VectorClockFromCap(varCap.WritesClock())
server.Log(v.UUId, "Restored", writeTxnId)
if result, err := disk.ReadonlyTransaction(func(rtxn *mdbs.RTxn) (interface{}, error) {
return db.ReadTxnFromDisk(rtxn, writeTxnId)
}).ResultError(); err == nil {
if result == nil || result.(*msgs.Txn) == nil {
panic(fmt.Sprintf("%v Unable to find txn %v on disk (%v)", v.UUId, writeTxnId, result))
}
actions := result.(*msgs.Txn).Actions()
v.curFrame = NewFrame(nil, v, writeTxnId, &actions, writeTxnClock, writesClock)
v.curFrameOnDisk = v.curFrame
} else {
return nil, err
}
v.varCap = &varCap
return v, nil
}
func (cr *connectionRun) handleMsgFromClient(msg *cmsgs.ClientMessage) error {
if cr.currentState != cr {
// probably just draining the queue from the reader after a restart
return nil
}
cr.missingBeats = 0
switch which := msg.Which(); which {
case cmsgs.CLIENTMESSAGE_HEARTBEAT:
// do nothing
case cmsgs.CLIENTMESSAGE_CLIENTTXNSUBMISSION:
ctxn := msg.ClientTxnSubmission()
origTxnId := common.MakeTxnId(ctxn.Id())
cr.submitter.SubmitClientTransaction(&ctxn, func(clientOutcome *cmsgs.ClientTxnOutcome, err error) {
switch {
case err != nil:
cr.clientTxnError(&ctxn, err, origTxnId)
case clientOutcome == nil: // shutdown
return
default:
seg := capn.NewBuffer(nil)
msg := cmsgs.NewRootClientMessage(seg)
msg.SetClientTxnOutcome(*clientOutcome)
cr.sendMessage(server.SegToBytes(msg.Segment))
}
})
default:
return cr.maybeRestartConnection(fmt.Errorf("Unexpected message type received from client: %v", which))
}
return nil
}
func (c *cache) updateFromTxnAbort(updates *msgs.ClientUpdate_List) []*common.VarUUId {
modifiedVars := make([]*common.VarUUId, 0, updates.Len())
c.Lock()
defer c.Unlock()
for idx, l := 0, updates.Len(); idx < l; idx++ {
update := updates.At(idx)
txnId := common.MakeTxnId(update.Version())
actions := update.Actions()
for idy, m := 0, actions.Len(); idy < m; idy++ {
action := actions.At(idy)
vUUId := common.MakeVarUUId(action.VarId())
//fmt.Printf("%v@%v ", vUUId, txnId)
switch action.Which() {
case msgs.CLIENTACTION_DELETE:
c.updateFromDelete(vUUId, txnId)
modifiedVars = append(modifiedVars, vUUId)
case msgs.CLIENTACTION_WRITE:
// We're missing TxnId and TxnId made a write of vUUId (to
// version TxnId).
write := action.Write()
refs := write.References()
if c.updateFromWrite(txnId, vUUId, write.Value(), &refs) {
modifiedVars = append(modifiedVars, vUUId)
}
default:
log.Fatal("Received update that was neither a read or write action:", action.Which())
}
}
}
//fmt.Println(".")
return modifiedVars
}
func (txn *Txn) submitRetryTransaction() error {
reads := make(map[common.VarUUId]*objectState)
for ancestor := txn; ancestor != nil; ancestor = ancestor.parent {
for _, obj := range ancestor.objs {
if _, found := reads[*obj.Id]; !found && obj.state.txn == ancestor && obj.state.read {
reads[*obj.Id] = obj.state
}
}
}
seg := capn.NewBuffer(nil)
cTxn := msgs.NewClientTxn(seg)
cTxn.SetRetry(true)
actions := msgs.NewClientActionList(seg, len(reads))
cTxn.SetActions(actions)
idx := 0
for _, state := range reads {
action := actions.At(idx)
action.SetVarId(state.Id[:])
action.SetRead()
action.Read().SetVersion(state.curVersion[:])
idx++
}
outcome, _, err := txn.conn.submitTransaction(&cTxn)
if err != nil {
return err
}
txn.stats.TxnId = common.MakeTxnId(outcome.FinalId())
for ancestor := txn; ancestor != nil; ancestor = ancestor.parent {
ancestor.resetInProgress = true
}
return nil
}
func TxnFromCap(exe *dispatcher.Executor, vd *VarDispatcher, stateChange TxnLocalStateChange, ourRMId common.RMId, txnCap *msgs.Txn) *Txn {
txnId := common.MakeTxnId(txnCap.Id())
actions := txnCap.Actions()
txn := &Txn{
Id: txnId,
Retry: txnCap.Retry(),
writes: make([]*common.VarUUId, 0, actions.Len()),
TxnCap: txnCap,
exe: exe,
vd: vd,
stateChange: stateChange,
}
allocations := txnCap.Allocations()
for idx, l := 0, allocations.Len(); idx < l; idx++ {
alloc := allocations.At(idx)
rmId := common.RMId(alloc.RmId())
if ourRMId == rmId {
txn.populate(alloc.ActionIndices(), actions)
break
}
}
return txn
}
func (cr *connectionRun) handleMsgFromPeer(msg *msgs.Message) error {
if cr.currentState != cr {
// probably just draining the queue from the reader after a restart
return nil
}
cr.missingBeats = 0
switch which := msg.Which(); which {
case msgs.MESSAGE_HEARTBEAT:
// do nothing
case msgs.MESSAGE_CLIENTTXNSUBMISSION:
ctxn := msg.ClientTxnSubmission()
origTxnId := common.MakeTxnId(ctxn.Id())
cr.submitter.SubmitClientTransaction(&ctxn, func(clientOutcome *msgs.ClientTxnOutcome, err error) {
switch {
case err != nil:
cr.clientTxnError(&ctxn, err, origTxnId)
case clientOutcome == nil: // shutdown
return
default:
seg := capn.NewBuffer(nil)
msg := msgs.NewRootMessage(seg)
msg.SetClientTxnOutcome(*clientOutcome)
cr.sendMessage(server.SegToBytes(msg.Segment))
}
})
default:
cr.connectionManager.Dispatchers.DispatchMessage(cr.remoteRMId, which, msg)
}
return nil
}
func (ad *AcceptorDispatcher) loadFromDisk(server *mdbs.MDBServer) {
res, err := server.ReadonlyTransaction(func(rtxn *mdbs.RTxn) (interface{}, error) {
return rtxn.WithCursor(db.DB.BallotOutcomes, func(cursor *mdb.Cursor) (interface{}, error) {
// cursor.Get returns a copy of the data. So it's fine for us
// to store and process this later - it's not about to be
// overwritten on disk.
count := 0
txnIdData, acceptorState, err := cursor.Get(nil, nil, mdb.FIRST)
for ; err == nil; txnIdData, acceptorState, err = cursor.Get(nil, nil, mdb.NEXT) {
count++
txnId := common.MakeTxnId(txnIdData)
acceptorStateCopy := acceptorState
ad.withAcceptorManager(txnId, func(am *AcceptorManager) {
am.loadFromData(txnId, acceptorStateCopy)
})
}
if err == mdb.NotFound {
// fine, we just fell off the end as expected.
return count, nil
} else {
return count, err
}
})
}).ResultError()
if err == nil {
log.Printf("Loaded %v acceptors from disk\n", res.(int))
} else {
log.Println("AcceptorDispatcher error loading from disk:", err)
}
}
func (bra *badReadAction) combine(action *msgs.Action, rmBal *rmBallot, txnId *common.TxnId, clockElem uint64) {
newActionType := action.Which()
braActionType := bra.action.Which()
switch {
case braActionType != msgs.ACTION_READ && newActionType != msgs.ACTION_READ:
// They're both writes in some way. Just order the txns
if clockElem > bra.clockElem || (clockElem == bra.clockElem && bra.txnId.Compare(txnId) == common.LT) {
bra.set(action, rmBal, txnId, clockElem)
}
case braActionType == msgs.ACTION_READ && newActionType == msgs.ACTION_READ:
braRead := bra.action.Read()
newRead := action.Read()
clockElem--
// If they read the same version, we really don't care.
if !bytes.Equal(braRead.Version(), newRead.Version()) {
// They read different versions, but which version was the latter?
if clockElem > bra.clockElem {
bra.set(action, rmBal, common.MakeTxnId(newRead.Version()), clockElem)
}
}
case braActionType == msgs.ACTION_READ:
if bytes.Equal(bra.txnId[:], txnId[:]) {
// The write will obviously be in the past of the
// existing read, but it's better to have the write
// as we can update the client with the actual
// value.
bra.set(action, rmBal, txnId, clockElem)
} else if clockElem > bra.clockElem {
// The write is after than the read
bra.set(action, rmBal, txnId, clockElem)
}
default: // Existing is not a read, but new is a read.
newRead := action.Read()
clockElem--
// If the read is a read of the existing write, better to keep the write
if !bytes.Equal(bra.txnId[:], newRead.Version()) {
if clockElem > bra.clockElem {
// The read must be of some value which was written after our existing write.
bra.set(action, rmBal, common.MakeTxnId(newRead.Version()), clockElem)
}
}
}
}
func (sts *SimpleTxnSubmitter) SubmitTransaction(txnCap *msgs.Txn, activeRMs []common.RMId, continuation TxnCompletionConsumer, delay time.Duration) {
seg := capn.NewBuffer(nil)
msg := msgs.NewRootMessage(seg)
msg.SetTxnSubmission(*txnCap)
txnId := common.MakeTxnId(txnCap.Id())
server.Log(txnId, "Submitting txn")
txnSender := paxos.NewRepeatingSender(server.SegToBytes(seg), activeRMs...)
var removeSenderCh chan server.EmptyStruct
if delay == 0 {
sts.connPub.AddServerConnectionSubscriber(txnSender)
} else {
removeSenderCh = make(chan server.EmptyStruct)
go func() {
// fmt.Printf("%v ", delay)
time.Sleep(delay)
sts.connPub.AddServerConnectionSubscriber(txnSender)
<-removeSenderCh
sts.connPub.RemoveServerConnectionSubscriber(txnSender)
}()
}
acceptors := paxos.GetAcceptorsFromTxn(txnCap)
shutdownFun := func(shutdown bool) {
delete(sts.outcomeConsumers, *txnId)
// fmt.Printf("sts%v ", len(sts.outcomeConsumers))
if delay == 0 {
sts.connPub.RemoveServerConnectionSubscriber(txnSender)
} else {
close(removeSenderCh)
}
// OSS is safe here - see above.
paxos.NewOneShotSender(paxos.MakeTxnSubmissionCompleteMsg(txnId), sts.connPub, acceptors...)
if shutdown {
if txnCap.Retry() {
// If this msg doesn't make it then proposers should
// observe our death and tidy up anyway. If it's just this
// connection shutting down then there should be no
// problem with these msgs getting to the propposers.
paxos.NewOneShotSender(paxos.MakeTxnSubmissionAbortMsg(txnId), sts.connPub, activeRMs...)
}
continuation(txnId, nil, nil)
}
}
shutdownFunPtr := &shutdownFun
sts.onShutdown[shutdownFunPtr] = server.EmptyStructVal
outcomeAccumulator := paxos.NewOutcomeAccumulator(int(txnCap.FInc()), acceptors)
consumer := func(sender common.RMId, txnId *common.TxnId, outcome *msgs.Outcome) {
if outcome, _ = outcomeAccumulator.BallotOutcomeReceived(sender, outcome); outcome != nil {
delete(sts.onShutdown, shutdownFunPtr)
shutdownFun(false)
continuation(txnId, outcome, nil)
}
}
sts.outcomeConsumers[*txnId] = consumer
// fmt.Printf("sts%v ", len(sts.outcomeConsumers))
}
func (vc versionCache) UpdateFromAbort(updates *msgs.Update_List) map[*msgs.Update][]*msgs.Action {
validUpdates := make(map[*msgs.Update][]*msgs.Action)
for idx, l := 0, updates.Len(); idx < l; idx++ {
update := updates.At(idx)
txnId := common.MakeTxnId(update.TxnId())
clock := eng.VectorClockFromCap(update.Clock())
actions := update.Actions()
validActions := make([]*msgs.Action, 0, actions.Len())
for idy, m := 0, actions.Len(); idy < m; idy++ {
action := actions.At(idy)
vUUId := common.MakeVarUUId(action.VarId())
clockElem := clock.Clock[*vUUId]
switch action.Which() {
case msgs.ACTION_MISSING:
if c, found := vc[*vUUId]; found {
cmp := c.txnId.Compare(txnId)
if clockElem > c.clockElem && cmp == common.EQ {
panic(fmt.Sprintf("Clock version increased on missing for %v@%v (%v > %v)", vUUId, txnId, clockElem, c.clockElem))
}
if clockElem > c.clockElem || (clockElem == c.clockElem && cmp == common.LT) {
delete(vc, *vUUId)
validActions = append(validActions, &action)
}
}
case msgs.ACTION_WRITE:
if c, found := vc[*vUUId]; found {
cmp := c.txnId.Compare(txnId)
if clockElem > c.clockElem && cmp == common.EQ {
panic(fmt.Sprintf("Clock version increased on write for %v@%v (%v > %v)", vUUId, txnId, clockElem, c.clockElem))
}
if clockElem > c.clockElem || (clockElem == c.clockElem && cmp == common.LT) {
c.txnId = txnId
c.clockElem = clockElem
validActions = append(validActions, &action)
}
} else {
vc[*vUUId] = &cached{
txnId: txnId,
clockElem: clockElem,
}
validActions = append(validActions, &action)
}
default:
panic(fmt.Sprintf("%v", action.Which()))
}
}
if len(validActions) != 0 {
validUpdates[&update] = validActions
}
}
return validUpdates
}
func (cm *ConnectionManager) DispatchMessage(sender common.RMId, msgType msgs.Message_Which, msg *msgs.Message) {
d := cm.Dispatchers
switch msgType {
case msgs.MESSAGE_TXNSUBMISSION:
txn := msg.TxnSubmission()
d.ProposerDispatcher.TxnReceived(sender, &txn)
case msgs.MESSAGE_SUBMISSIONOUTCOME:
outcome := msg.SubmissionOutcome()
txnId := common.MakeTxnId(outcome.Txn().Id())
connNumber := binary.BigEndian.Uint32(txnId[8:12])
bootNumber := binary.BigEndian.Uint32(txnId[12:16])
if conn := cm.GetClient(bootNumber, connNumber); conn == nil {
// OSS is safe here - it's the default action on receipt of outcome for unknown client.
paxos.NewOneShotSender(paxos.MakeTxnSubmissionCompleteMsg(txnId), cm, sender)
} else {
conn.SubmissionOutcomeReceived(sender, txnId, &outcome)
return
}
case msgs.MESSAGE_SUBMISSIONCOMPLETE:
tsc := msg.SubmissionComplete()
d.AcceptorDispatcher.TxnSubmissionCompleteReceived(sender, &tsc)
case msgs.MESSAGE_SUBMISSIONABORT:
tsa := msg.SubmissionAbort()
d.ProposerDispatcher.TxnSubmissionAbortReceived(sender, &tsa)
case msgs.MESSAGE_ONEATXNVOTES:
oneATxnVotes := msg.OneATxnVotes()
d.AcceptorDispatcher.OneATxnVotesReceived(sender, &oneATxnVotes)
case msgs.MESSAGE_ONEBTXNVOTES:
oneBTxnVotes := msg.OneBTxnVotes()
d.ProposerDispatcher.OneBTxnVotesReceived(sender, &oneBTxnVotes)
case msgs.MESSAGE_TWOATXNVOTES:
twoATxnVotes := msg.TwoATxnVotes()
d.AcceptorDispatcher.TwoATxnVotesReceived(sender, &twoATxnVotes)
case msgs.MESSAGE_TWOBTXNVOTES:
twoBTxnVotes := msg.TwoBTxnVotes()
d.ProposerDispatcher.TwoBTxnVotesReceived(sender, &twoBTxnVotes)
case msgs.MESSAGE_TXNLOCALLYCOMPLETE:
tlc := msg.TxnLocallyComplete()
d.AcceptorDispatcher.TxnLocallyCompleteReceived(sender, &tlc)
case msgs.MESSAGE_TXNGLOBALLYCOMPLETE:
tgc := msg.TxnGloballyComplete()
d.ProposerDispatcher.TxnGloballyCompleteReceived(sender, &tgc)
case msgs.MESSAGE_TOPOLOGYCHANGEREQUEST:
// do nothing - we've just sent it to ourselves.
case msgs.MESSAGE_MIGRATION:
migration := msg.Migration()
cm.Transmogrifier.MigrationReceived(sender, &migration)
case msgs.MESSAGE_MIGRATIONCOMPLETE:
migrationComplete := msg.MigrationComplete()
cm.Transmogrifier.MigrationCompleteReceived(sender, &migrationComplete)
default:
panic(fmt.Sprintf("Unexpected message received from %v (%v)", sender, msgType))
}
}
func (pd *ProposerDispatcher) ImmigrationReceived(migration *msgs.Migration, stateChange eng.TxnLocalStateChange) {
elemsList := migration.Elems()
elemsCount := elemsList.Len()
for idx := 0; idx < elemsCount; idx++ {
elem := elemsList.At(idx)
txnCap := elem.Txn()
txnId := common.MakeTxnId(txnCap.Id())
varCaps := elem.Vars()
pd.withProposerManager(txnId, func(pm *ProposerManager) { pm.ImmigrationReceived(txnId, &txnCap, &varCaps, stateChange) })
}
}
func loadVars(disk *mdbs.MDBServer, vars map[common.VarUUId]*varstate) {
_, err := disk.ReadonlyTransaction(func(rtxn *mdbs.RTxn) (interface{}, error) {
return rtxn.WithCursor(db.DB.Vars, func(cursor *mdb.Cursor) (interface{}, error) {
key, data, err := cursor.Get(nil, nil, mdb.FIRST)
for ; err == nil; key, data, err = cursor.Get(nil, nil, mdb.NEXT) {
vUUId := common.MakeVarUUId(key)
seg, _, err := capn.ReadFromMemoryZeroCopy(data)
if err != nil {
log.Println(err)
continue
}
varCap := msgs.ReadRootVar(seg)
pos := varCap.Positions()
positions := (*common.Positions)(&pos)
writeTxnId := common.MakeTxnId(varCap.WriteTxnId())
writeTxnClock := eng.VectorClockFromCap(varCap.WriteTxnClock())
writesClock := eng.VectorClockFromCap(varCap.WritesClock())
if state, found := vars[*vUUId]; found {
if err := state.matches(disk, writeTxnId, writeTxnClock, writesClock, positions); err != nil {
log.Println(err)
}
} else {
state = &varstate{
vUUId: vUUId,
disks: []*mdbs.MDBServer{disk},
writeTxnId: writeTxnId,
writeTxnClock: writeTxnClock,
writeWritesClock: writesClock,
positions: positions,
}
vars[*vUUId] = state
}
}
if err == mdb.NotFound {
return nil, nil
} else {
return nil, err
}
})
}).ResultError()
if err != nil {
log.Println(err)
}
}
func (sts *SimpleTxnSubmitter) SubmitTransaction(txnCap *msgs.Txn, activeRMs []common.RMId, continuation TxnCompletionConsumer, delay time.Duration) {
seg := capn.NewBuffer(nil)
msg := msgs.NewRootMessage(seg)
msg.SetTxnSubmission(*txnCap)
txnId := common.MakeTxnId(txnCap.Id())
server.Log(txnId, "Submitting txn")
txnSender := paxos.NewRepeatingSender(server.SegToBytes(seg), activeRMs...)
if delay == 0 {
sts.connectionManager.AddSender(txnSender)
} else {
go func() {
// fmt.Printf("%v ", delay)
time.Sleep(delay)
sts.connectionManager.AddSender(txnSender)
}()
}
acceptors := paxos.GetAcceptorsFromTxn(txnCap)
shutdownFun := func(shutdown bool) {
delete(sts.outcomeConsumers, *txnId)
// fmt.Printf("sts%v ", len(sts.outcomeConsumers))
sts.connectionManager.RemoveSenderAsync(txnSender)
paxos.NewOneShotSender(paxos.MakeTxnSubmissionCompleteMsg(txnId), sts.connectionManager, acceptors...)
if shutdown {
if txnCap.Retry() {
paxos.NewOneShotSender(paxos.MakeTxnSubmissionAbortMsg(txnId), sts.connectionManager, activeRMs...)
}
continuation(txnId, nil)
}
}
shutdownFunPtr := &shutdownFun
sts.onShutdown[shutdownFunPtr] = server.EmptyStructVal
outcomeAccumulator := paxos.NewOutcomeAccumulator(int(txnCap.FInc()), acceptors)
consumer := func(sender common.RMId, txnId *common.TxnId, outcome *msgs.Outcome) {
if outcome, _ = outcomeAccumulator.BallotOutcomeReceived(sender, outcome); outcome != nil {
delete(sts.onShutdown, shutdownFunPtr)
shutdownFun(false)
continuation(txnId, outcome)
}
}
sts.outcomeConsumers[*txnId] = consumer
// fmt.Printf("sts%v ", len(sts.outcomeConsumers))
}
func (d *Dispatchers) DispatchMessage(sender common.RMId, msgType msgs.Message_Which, msg *msgs.Message) {
switch msgType {
case msgs.MESSAGE_TXNSUBMISSION:
txn := msg.TxnSubmission()
d.ProposerDispatcher.TxnReceived(sender, &txn)
case msgs.MESSAGE_SUBMISSIONOUTCOME:
outcome := msg.SubmissionOutcome()
txnId := common.MakeTxnId(outcome.Txn().Id())
connNumber := binary.BigEndian.Uint32(txnId[8:12])
bootNumber := binary.BigEndian.Uint32(txnId[12:16])
if conn := d.connectionManager.GetClient(bootNumber, connNumber); conn == nil {
NewOneShotSender(MakeTxnSubmissionCompleteMsg(txnId), d.connectionManager, sender)
} else {
conn.SubmissionOutcomeReceived(sender, txnId, &outcome)
return
}
case msgs.MESSAGE_SUBMISSIONCOMPLETE:
tsc := msg.SubmissionComplete()
d.AcceptorDispatcher.TxnSubmissionCompleteReceived(sender, &tsc)
case msgs.MESSAGE_SUBMISSIONABORT:
tsa := msg.SubmissionAbort()
d.ProposerDispatcher.TxnSubmissionAbortReceived(sender, &tsa)
case msgs.MESSAGE_ONEATXNVOTES:
oneATxnVotes := msg.OneATxnVotes()
d.AcceptorDispatcher.OneATxnVotesReceived(sender, &oneATxnVotes)
case msgs.MESSAGE_ONEBTXNVOTES:
oneBTxnVotes := msg.OneBTxnVotes()
d.ProposerDispatcher.OneBTxnVotesReceived(sender, &oneBTxnVotes)
case msgs.MESSAGE_TWOATXNVOTES:
twoATxnVotes := msg.TwoATxnVotes()
d.AcceptorDispatcher.TwoATxnVotesReceived(sender, &twoATxnVotes)
case msgs.MESSAGE_TWOBTXNVOTES:
twoBTxnVotes := msg.TwoBTxnVotes()
d.ProposerDispatcher.TwoBTxnVotesReceived(sender, &twoBTxnVotes)
case msgs.MESSAGE_TXNLOCALLYCOMPLETE:
tlc := msg.TxnLocallyComplete()
d.AcceptorDispatcher.TxnLocallyCompleteReceived(sender, &tlc)
case msgs.MESSAGE_TXNGLOBALLYCOMPLETE:
tgc := msg.TxnGloballyComplete()
d.ProposerDispatcher.TxnGloballyCompleteReceived(sender, &tgc)
default:
panic(fmt.Sprintf("Unexpected message received from %v", sender))
}
}
func (pd *ProposerDispatcher) loadFromDisk(db *db.Databases) {
res, err := db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
res, _ := rtxn.WithCursor(db.Proposers, func(cursor *mdbs.Cursor) interface{} {
// cursor.Get returns a copy of the data. So it's fine for us
// to store and process this later - it's not about to be
// overwritten on disk.
proposerStates := make(map[*common.TxnId][]byte)
txnIdData, proposerState, err := cursor.Get(nil, nil, mdb.FIRST)
for ; err == nil; txnIdData, proposerState, err = cursor.Get(nil, nil, mdb.NEXT) {
txnId := common.MakeTxnId(txnIdData)
proposerStates[txnId] = proposerState
}
if err == mdb.NotFound {
// fine, we just fell off the end as expected.
return proposerStates
} else {
cursor.Error(err)
return nil
}
})
return res
}).ResultError()
if err != nil {
panic(fmt.Sprintf("ProposerDispatcher error loading from disk: %v", err))
} else if res != nil {
proposerStates := res.(map[*common.TxnId][]byte)
for txnId, proposerState := range proposerStates {
proposerStateCopy := proposerState
txnIdCopy := txnId
pd.withProposerManager(txnIdCopy, func(pm *ProposerManager) {
if err := pm.loadFromData(txnIdCopy, proposerStateCopy); err != nil {
log.Printf("ProposerDispatcher error loading %v from disk: %v\n", txnIdCopy, err)
}
})
}
log.Printf("Loaded %v proposers from disk\n", len(proposerStates))
}
}
func VarFromData(data []byte, exe *dispatcher.Executor, db *db.Databases, vm *VarManager) (*Var, error) {
seg, _, err := capn.ReadFromMemoryZeroCopy(data)
if err != nil {
return nil, err
}
varCap := msgs.ReadRootVar(seg)
v := newVar(common.MakeVarUUId(varCap.Id()), exe, db, vm)
positions := varCap.Positions()
if positions.Len() != 0 {
v.positions = (*common.Positions)(&positions)
}
writeTxnId := common.MakeTxnId(varCap.WriteTxnId())
writeTxnClock := VectorClockFromCap(varCap.WriteTxnClock())
writesClock := VectorClockFromCap(varCap.WritesClock())
server.Log(v.UUId, "Restored", writeTxnId)
if result, err := db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
return db.ReadTxnBytesFromDisk(rtxn, writeTxnId)
}).ResultError(); err == nil && result != nil {
bites := result.([]byte)
if seg, _, err := capn.ReadFromMemoryZeroCopy(bites); err == nil {
txn := msgs.ReadRootTxn(seg)
actions := txn.Actions()
v.curFrame = NewFrame(nil, v, writeTxnId, &actions, writeTxnClock, writesClock)
v.curFrameOnDisk = v.curFrame
v.varCap = &varCap
return v, nil
} else {
return nil, err
}
} else {
return nil, err
}
}
func (pd *ProposerDispatcher) TxnSubmissionAbortReceived(sender common.RMId, tsa *msgs.TxnSubmissionAbort) {
txnId := common.MakeTxnId(tsa.TxnId())
pd.withProposerManager(txnId, func(pm *ProposerManager) { pm.TxnSubmissionAbortReceived(sender, txnId) })
}
func (pd *ProposerDispatcher) TxnGloballyCompleteReceived(sender common.RMId, tgc *msgs.TxnGloballyComplete) {
txnId := common.MakeTxnId(tgc.TxnId())
pd.withProposerManager(txnId, func(pm *ProposerManager) { pm.TxnGloballyCompleteReceived(sender, txnId) })
}
func (pd *ProposerDispatcher) OneBTxnVotesReceived(sender common.RMId, oneBTxnVotes *msgs.OneBTxnVotes) {
txnId := common.MakeTxnId(oneBTxnVotes.TxnId())
pd.withProposerManager(txnId, func(pm *ProposerManager) { pm.OneBTxnVotesReceived(sender, txnId, oneBTxnVotes) })
}
func (pd *ProposerDispatcher) TxnReceived(sender common.RMId, txn *msgs.Txn) {
txnId := common.MakeTxnId(txn.Id())
pd.withProposerManager(txnId, func(pm *ProposerManager) { pm.TxnReceived(sender, txnId, txn) })
}
func (txn *Txn) populate(actionIndices capn.UInt16List, actions msgs.Action_List) {
localActions := make([]localAction, actionIndices.Len())
txn.localActions = localActions
var action *localAction
actionIndicesIdx := 0
actionIndex := -1
if actionIndicesIdx < actionIndices.Len() {
actionIndex = int(actionIndices.At(actionIndicesIdx))
action = &localActions[actionIndicesIdx]
}
for idx, l := 0, actions.Len(); idx < l; idx++ {
actionCap := actions.At(idx)
if idx == actionIndex {
action.Txn = txn
action.vUUId = common.MakeVarUUId(actionCap.VarId())
}
switch actionCap.Which() {
case msgs.ACTION_READ:
if idx == actionIndex {
readCap := actionCap.Read()
readVsn := common.MakeTxnId(readCap.Version())
action.readVsn = readVsn
}
case msgs.ACTION_WRITE:
if idx == actionIndex {
action.writeTxnActions = &actions
action.writeAction = &actionCap
txn.writes = append(txn.writes, action.vUUId)
} else {
txn.writes = append(txn.writes, common.MakeVarUUId(actionCap.VarId()))
}
case msgs.ACTION_READWRITE:
if idx == actionIndex {
readWriteCap := actionCap.Readwrite()
readVsn := common.MakeTxnId(readWriteCap.Version())
action.readVsn = readVsn
action.writeTxnActions = &actions
action.writeAction = &actionCap
txn.writes = append(txn.writes, action.vUUId)
} else {
txn.writes = append(txn.writes, common.MakeVarUUId(actionCap.VarId()))
}
case msgs.ACTION_CREATE:
if idx == actionIndex {
createCap := actionCap.Create()
positions := common.Positions(createCap.Positions())
action.writeTxnActions = &actions
action.writeAction = &actionCap
action.createPositions = &positions
txn.writes = append(txn.writes, action.vUUId)
} else {
txn.writes = append(txn.writes, common.MakeVarUUId(actionCap.VarId()))
}
case msgs.ACTION_ROLL:
if idx == actionIndex {
rollCap := actionCap.Roll()
readVsn := common.MakeTxnId(rollCap.Version())
action.readVsn = readVsn
action.writeTxnActions = &actions
action.writeAction = &actionCap
action.roll = true
txn.writes = append(txn.writes, action.vUUId)
} else {
txn.writes = append(txn.writes, common.MakeVarUUId(actionCap.VarId()))
}
default:
panic(fmt.Sprintf("Unexpected action type: %v", actionCap.Which()))
}
if idx == actionIndex {
actionIndicesIdx++
if actionIndicesIdx < actionIndices.Len() {
actionIndex = int(actionIndices.At(actionIndicesIdx))
action = &localActions[actionIndicesIdx]
}
}
}
if actionIndicesIdx != actionIndices.Len() {
panic(fmt.Sprintf("Expected to find %v local actions, but only found %v", actionIndices.Len(), actionIndicesIdx))
}
}
func (lc *locationChecker) locationCheck(cell *varWrapperCell) error {
vUUId := cell.vUUId
varCap := cell.varCap
foundIn := cell.store
fmt.Printf("%v %v\n", foundIn, vUUId)
txnId := common.MakeTxnId(varCap.WriteTxnId())
res, err := foundIn.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
return foundIn.db.ReadTxnBytesFromDisk(rtxn, txnId)
}).ResultError()
if err != nil {
return err
}
txnBites, ok := res.([]byte)
if res == nil || (ok && txnBites == nil) {
return fmt.Errorf("Failed to find %v from %v in %v", txnId, vUUId, foundIn)
}
seg, _, err := capn.ReadFromMemoryZeroCopy(txnBites)
if err != nil {
return err
}
txnCap := msgs.ReadRootTxn(seg)
positions := varCap.Positions().ToArray()
rmIds, err := lc.resolver.ResolveHashCodes(positions)
if err != nil {
return err
}
foundLocal := false
for _, rmId := range rmIds {
if foundLocal = rmId == foundIn.rmId; foundLocal {
break
}
}
if !foundLocal {
// It must have emigrated but we don't delete.
txnId = nil
}
for _, rmId := range rmIds {
if rmId == foundIn.rmId {
continue
} else if remote, found := lc.stores[rmId]; found {
res, err := remote.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
bites, err := rtxn.Get(remote.db.Vars, vUUId[:])
if err == mdb.NotFound {
return nil
} else if err == nil {
return bites
} else {
return nil
}
}).ResultError()
if err != nil {
return err
}
varBites, ok := res.([]byte)
if res == nil || (ok && varBites == nil) {
if vUUId.BootCount() == 1 && vUUId.ConnectionCount() == 0 &&
(txnId == nil ||
(txnId.BootCount() == 1 && txnId.ConnectionCount() == 0 &&
txnCap.Actions().Len() == 1 && txnCap.Actions().At(0).Which() == msgs.ACTION_CREATE)) {
fmt.Printf("Failed to find %v in %v (%v, %v, %v) but it looks like it's a bad root.\n", vUUId, remote, rmIds, positions, foundIn)
} else {
return fmt.Errorf("Failed to find %v in %v (%v, %v, %v)", vUUId, remote, rmIds, positions, foundIn)
}
} else {
seg, _, err := capn.ReadFromMemoryZeroCopy(varBites)
if err != nil {
return err
}
remoteTxnId := common.MakeTxnId(msgs.ReadRootVar(seg).WriteTxnId())
if txnId == nil {
txnId = remoteTxnId
}
if remoteTxnId.Compare(txnId) != common.EQ {
return fmt.Errorf("%v on %v is at %v; on %v is at %v", vUUId, foundIn, txnId, remote, remoteTxnId)
}
}
}
}
return nil
}
func (s *store) LoadTopology() error {
res, err := s.db.ReadonlyTransaction(func(rtxn *mdbs.RTxn) interface{} {
bites, err := rtxn.Get(s.db.Vars, configuration.TopologyVarUUId[:])
if err != nil {
rtxn.Error(err)
return nil
}
seg, _, err := capn.ReadFromMemoryZeroCopy(bites)
if err != nil {
rtxn.Error(err)
return nil
}
varCap := msgs.ReadRootVar(seg)
txnId := common.MakeTxnId(varCap.WriteTxnId())
bites = s.db.ReadTxnBytesFromDisk(rtxn, txnId)
if bites == nil {
rtxn.Error(fmt.Errorf("Unable to find txn for topology: %v", txnId))
return nil
}
seg, _, err = capn.ReadFromMemoryZeroCopy(bites)
if err != nil {
rtxn.Error(err)
return nil
}
txnCap := msgs.ReadRootTxn(seg)
actions := txnCap.Actions()
if actions.Len() != 1 {
rtxn.Error(fmt.Errorf("Topology txn has %v actions; expected 1", actions.Len()))
return nil
}
action := actions.At(0)
var refs msgs.VarIdPos_List
switch action.Which() {
case msgs.ACTION_WRITE:
w := action.Write()
bites = w.Value()
refs = w.References()
case msgs.ACTION_READWRITE:
rw := action.Readwrite()
bites = rw.Value()
refs = rw.References()
case msgs.ACTION_CREATE:
c := action.Create()
bites = c.Value()
refs = c.References()
default:
rtxn.Error(fmt.Errorf("Expected topology txn action to be w, rw, or c; found %v", action.Which()))
return nil
}
if refs.Len() != 1 {
rtxn.Error(fmt.Errorf("Topology txn action has %v references; expected 1", refs.Len()))
return nil
}
rootRef := refs.At(0)
seg, _, err = capn.ReadFromMemoryZeroCopy(bites)
if err != nil {
rtxn.Error(err)
return nil
}
topology, err := configuration.TopologyFromCap(txnId, &rootRef, bites)
if err != nil {
rtxn.Error(err)
return nil
}
return topology
}).ResultError()
if err != nil {
return err
}
s.topology = res.(*configuration.Topology)
return nil
}
package server
import (
"fmt"
capn "github.com/glycerine/go-capnproto"
"goshawkdb.io/common"
msgs "goshawkdb.io/common/capnp"
"goshawkdb.io/server/configuration"
)
var (
TopologyVarUUId = common.MakeVarUUId([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0})
VersionOne = common.MakeTxnId([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1})
)
var BlankTopology = &Topology{
Configuration: &configuration.Configuration{
F: 0,
MaxRMCount: 1,
},
AllRMs: []common.RMId{},
FInc: 1,
TwoFInc: 1,
DBVersion: VersionOne,
}
type Topology struct {
*configuration.Configuration
AllRMs common.RMIds
FInc uint8
TwoFInc uint16
func (instId instanceId) String() string {
txnId := common.MakeTxnId(instId[0:common.KeyLen])
rmId := common.RMId(binary.BigEndian.Uint32(instId[common.KeyLen:]))
vUUId := common.MakeVarUUId(instId[common.KeyLen+4:])
return fmt.Sprintf("PaxosInstanceId:%v:%v:%v", txnId, rmId, vUUId)
}