func livetestlibSetupEasyTimeoutCluster(clusterSize int) []*replica.Replica {
nodes := make([]*replica.Replica, clusterSize)
for i := range nodes {
param := &replica.Param{
ExecuteInterval: time.Second * 50, // disable execution
TimeoutInterval: time.Millisecond * 20, // disable timeout
ReplicaId: uint8(i),
Size: uint8(clusterSize),
StateMachine: new(test.DummySM),
Transporter: transporter.NewDummyTR(uint8(i), clusterSize),
PersistentPath: tempfile(),
}
nodes[i], _ = replica.New(param)
}
chs := make([]chan message.Message, clusterSize)
for i := range nodes {
chs[i] = nodes[i].MessageChan
}
for i := range nodes {
nodes[i].Transporter.(*transporter.DummyTransporter).RegisterChannels(chs)
nodes[i].Start()
}
return nodes
}
// test the correctness of the propose id without batching
func TestProposeIdNoBatch(t *testing.T) {
N := 5000
param := &Param{
ReplicaId: 0,
Size: 5,
StateMachine: new(test.DummySM),
EnableBatching: false,
Transporter: transporter.NewDummyTR(0, 5),
PersistentPath: tempfile(),
}
r, _ := New(param)
// only start the propose
go r.proposeLoop()
go r.eventLoop()
defer close(r.stop)
resultIDs := make([]chan uint64, N)
expectIDs := make([]uint64, N)
j := uint64(0)
for i := 0; i < N; i++ {
if r.IsCheckpoint(j) {
j++
}
resultIDs[i] = r.Propose(message.Command("hello"))
expectIDs[i] = j
j++
}
for i := 0; i < N; i++ {
assert.Equal(t, <-resultIDs[i], expectIDs[i])
}
}
func TestStoreRestoreSinglePreparingInstance(t *testing.T) {
param := &Param{
ReplicaId: 0,
Size: 3,
StateMachine: new(test.DummySM),
EnableBatching: true,
BatchInterval: time.Millisecond * 50,
Transporter: transporter.NewDummyTR(0, 5),
PersistentPath: tempfile(),
}
r, err := New(param)
assert.NoError(t, err)
defer func() {
r.store.Close()
r.store.Drop()
}()
inst := commonTestlibExamplePreparingInstance()
assert.NoError(t, r.StoreSingleInstance(inst))
storedInst, err := r.RestoreSingleInstance(inst.rowId, inst.id)
assert.NoError(t, err)
assert.Equal(t, storedInst.cmds, inst.cmds)
assert.Equal(t, storedInst.deps, inst.deps)
assert.Equal(t, storedInst.status, inst.status)
assert.Equal(t, storedInst.ballot, inst.ballot)
assert.Equal(t, storedInst.recoveryInfo.ballot, inst.recoveryInfo.ballot)
assert.Equal(t, storedInst.recoveryInfo.cmds, inst.recoveryInfo.cmds)
assert.Equal(t, storedInst.recoveryInfo.deps, inst.recoveryInfo.deps)
assert.Equal(t, storedInst.recoveryInfo.status, inst.recoveryInfo.status)
assert.Equal(t, storedInst.recoveryInfo.formerStatus, inst.recoveryInfo.formerStatus)
assert.Equal(t, storedInst.executed, inst.executed)
}
func TestNewReplica(t *testing.T) {
param := &Param{
ReplicaId: 3,
Size: 5,
StateMachine: new(test.DummySM),
Transporter: transporter.NewDummyTR(3, 5),
PersistentPath: tempfile(),
}
r, _ := New(param)
assert.True(t, r.Id == 3)
assert.True(t, r.Size == 5)
assert.True(t, len(r.MaxInstanceNum) == 5)
assert.True(t, len(r.InstanceMatrix) == 5)
assert.Equal(t, r.StateMachine, new(test.DummySM))
assert.True(t, r.Epoch == 1)
for i := range r.InstanceMatrix {
assert.True(t, len(r.InstanceMatrix[i]) == defaultInstancesLength)
}
param.Size = 4
_, err := New(param)
assert.NotNil(t, err)
}
func TestMakeInitialBallot(t *testing.T) {
param := &Param{
ReplicaId: 3,
Size: 5,
StateMachine: new(test.DummySM),
Transporter: transporter.NewDummyTR(3, 5),
PersistentPath: tempfile(),
}
r, _ := New(param)
r.Epoch = 3
b := r.makeInitialBallot()
assert.Equal(t, b, message.NewBallot(3, 0, 3))
}
func TestStoreRestoreMultipleInstances(t *testing.T) {
param := &Param{
ReplicaId: 0,
Size: 3,
StateMachine: new(test.DummySM),
EnableBatching: true,
BatchInterval: time.Millisecond * 50,
Transporter: transporter.NewDummyTR(0, 5),
PersistentPath: tempfile(),
}
r, err := New(param)
assert.NoError(t, err)
defer func() {
r.store.Close()
r.store.Drop()
}()
instGroup := make([]*Instance, 5)
instGroup[0] = commonTestlibExampleNilStatusInstance()
instGroup[0].id = 1
instGroup[1] = commonTestlibExamplePreAcceptedInstance()
instGroup[1].id = 2
instGroup[2] = commonTestlibExampleAcceptedInstance()
instGroup[2].id = 3
instGroup[3] = commonTestlibExampleCommittedInstance()
instGroup[3].id = 4
instGroup[4] = commonTestlibExamplePreparingInstance()
instGroup[4].id = 5
assert.NoError(t, r.StoreInstances(instGroup...))
for _, inst := range instGroup {
storedInst, err := r.RestoreSingleInstance(inst.rowId, inst.id)
assert.NoError(t, err)
assert.Equal(t, storedInst.cmds, inst.cmds)
assert.Equal(t, storedInst.deps, inst.deps)
assert.Equal(t, storedInst.status, inst.status)
assert.Equal(t, storedInst.ballot, inst.ballot)
if inst.isAtStatus(preparing) {
assert.Equal(t, storedInst.recoveryInfo.ballot, inst.recoveryInfo.ballot)
assert.Equal(t, storedInst.recoveryInfo.cmds, inst.recoveryInfo.cmds)
assert.Equal(t, storedInst.recoveryInfo.deps, inst.recoveryInfo.deps)
assert.Equal(t, storedInst.recoveryInfo.status, inst.recoveryInfo.status)
assert.Equal(t, storedInst.recoveryInfo.formerStatus, inst.recoveryInfo.formerStatus)
assert.Equal(t, storedInst.executed, inst.executed)
}
}
}
// return a replica with id=5, size=5, and maxinstancenum of [1,2,3,4,5]
func depsTestSetupReplica() (r *Replica, i *Instance) {
param := &Param{
ReplicaId: 4,
Size: 5,
StateMachine: new(test.DummySM),
Transporter: transporter.NewDummyTR(4, 5),
PersistentPath: tempfile(),
}
r, _ = New(param)
for i := 0; i < 5; i++ {
r.MaxInstanceNum[i] = uint64(conflictNotFound + 1 + uint64(i))
instance := NewInstance(r, r.Id, conflictNotFound+1+uint64(i))
instance.cmds = commonTestlibExampleCommands().Clone()
r.InstanceMatrix[i][instance.id] = instance
}
i = NewInstance(r, r.Id, 6)
return
}
func TestStoreAndRestoreReplica(t *testing.T) {
param := &Param{
ReplicaId: 0,
Size: 3,
StateMachine: new(test.DummySM),
EnableBatching: true,
BatchInterval: time.Millisecond * 50,
Transporter: transporter.NewDummyTR(0, 5),
PersistentPath: tempfile(),
}
r, err := New(param)
assert.NoError(t, err)
r.MaxInstanceNum[0] = 42
r.MaxInstanceNum[1] = 88
r.MaxInstanceNum[2] = 102
r.ExecutedUpTo[0] = 42
r.ExecutedUpTo[1] = 88
r.ExecutedUpTo[2] = 33
r.ProposeNum = 10
// store to disk
assert.NoError(t, r.StoreReplica())
// Shutdown replica -- BoltDB enforces one process per database
// In fact I have no idea idea how this worked with LevelDB
r.Stop()
// restore from disk
param.Restore = true
rr, err := New(param)
assert.NoError(t, err)
assert.Equal(t, r.Id, rr.Id)
assert.Equal(t, r.Size, rr.Size)
assert.Equal(t, r.MaxInstanceNum, rr.MaxInstanceNum)
assert.Equal(t, r.ExecutedUpTo, rr.ExecutedUpTo)
assert.Equal(t, r.ProposeNum, rr.ProposeNum)
r.store.Drop()
rr.store.Drop()
}
// test persistent store
func TestStoreSingleInstance(t *testing.T) {
param := &Param{
ReplicaId: 0,
Size: 3,
StateMachine: new(test.DummySM),
EnableBatching: true,
BatchInterval: time.Millisecond * 50,
Transporter: transporter.NewDummyTR(0, 5),
PersistentPath: tempfile(),
}
r, err := New(param)
assert.NoError(t, err)
defer func() {
r.store.Close()
r.store.Drop()
}()
inst := commonTestlibExampleAcceptedInstance()
assert.NoError(t, r.StoreSingleInstance(inst))
}
// test the correctness of the propose id with batching
func TestProposeIdWithBatch(t *testing.T) {
N := 5000
B := 100
param := &Param{
ReplicaId: 0,
Size: 5,
StateMachine: new(test.DummySM),
EnableBatching: true,
BatchInterval: time.Millisecond * 50,
Transporter: transporter.NewDummyTR(0, 5),
PersistentPath: tempfile(),
}
r, _ := New(param)
// only start the propose
go r.proposeLoop()
go r.eventLoop()
defer close(r.stop)
resultIDs := make([]chan uint64, N)
expectIDs := make([]uint64, N)
// let's batch 100 commands in a group
exp := uint64(0)
for i := 0; i < N/B; i++ {
for j := 0; j < B; j++ {
if r.IsCheckpoint(exp) {
exp++
}
resultIDs[i*B+j] = r.Propose(message.Command("hello"))
expectIDs[i*B+j] = exp
}
time.Sleep(param.BatchInterval * 2)
exp++
}
for i := 0; i < N; i++ {
assert.Equal(t, <-resultIDs[i], expectIDs[i])
}
}