// goSchedule waits for timeouts to complete and runs the alarms.
func (this *Alarm) goSchedule() {
defer this.wg.Done()
this.mutex.Lock()
defer this.mutex.Unlock()
for {
for len(this.timestampList) == 0 {
if err := this.cond.Wait(); errs.IsClosed(err) {
return
}
}
now := time.Now()
timestamp := this.timestampList[0]
if now.After(timestamp) {
pendingList := this.pendingMap[timestamp]
delete(this.pendingMap, timestamp)
numTimestamps := len(this.timestampList)
for ii := 1; ii < numTimestamps; ii++ {
this.timestampList[ii-1] = this.timestampList[ii]
}
this.timestampList = this.timestampList[:numTimestamps-1]
jobList := make([]func() error, len(pendingList))
for ii, uid := range pendingList {
jobList[ii] = this.jobMap[uid]
delete(this.jobMap, uid)
}
this.mutex.Unlock()
for _, fn := range jobList {
fn()
}
this.mutex.Lock()
continue
}
timeout := timestamp.Sub(now)
timeoutCh := time.After(timeout)
if err := this.cond.WaitTimeout(timeoutCh); errs.IsClosed(err) {
return
}
}
}
func TestCondition(test *testing.T) {
mutex := sync.Mutex{}
cond := &Condition{}
cond.Initialize(&mutex)
// A signal must wake up waiting condition.
mutex.Lock()
go func() {
mutex.Lock()
cond.Signal()
mutex.Unlock()
}()
if err := cond.Wait(); err != nil {
test.Errorf("condition woke up with a non-nil error: %v", err)
return
}
mutex.Unlock()
// A signal must wake up timed wait.
mutex.Lock()
go func() {
mutex.Lock()
cond.Signal()
mutex.Unlock()
}()
if err := cond.WaitTimeout(time.After(time.Second)); err != nil {
test.Errorf("timed wait woke up with a non-nil error on signal: %v", err)
return
}
mutex.Unlock()
// A broadcast must wake up all waiters.
wg := sync.WaitGroup{}
for ii := 0; ii < 10; ii++ {
doneCh := make(chan bool)
wg.Add(1)
go func() {
mutex.Lock()
doneCh <- true
if err := cond.Wait(); err != nil {
test.Errorf("wait returned with unexpected status: %v", err)
}
mutex.Unlock()
wg.Done()
}()
<-doneCh
}
mutex.Lock()
cond.Broadcast()
mutex.Unlock()
wg.Wait()
// A broadcast must wake up all timed waiters.
for ii := 0; ii < 10; ii++ {
doneCh := make(chan bool)
wg.Add(1)
go func() {
mutex.Lock()
doneCh <- true
if err := cond.WaitTimeout(time.After(time.Minute)); err != nil {
test.Errorf("timed wait returned with unexpected status: %v", err)
}
mutex.Unlock()
wg.Done()
}()
<-doneCh
}
mutex.Lock()
cond.Broadcast()
mutex.Unlock()
wg.Wait()
// A timeout must wake up the condition.
mutex.Lock()
if err := cond.WaitTimeout(time.After(time.Second)); !errs.IsTimeout(err) {
test.Errorf("timed wait woke up with unexpected status: %v", err)
return
}
mutex.Unlock()
// Closing a condition must wake up all timed waiters and normal waiters.
for ii := 0; ii < 10; ii++ {
doneCh := make(chan bool)
wg.Add(1)
go func() {
mutex.Lock()
doneCh <- true
if err := cond.Wait(); !errs.IsClosed(err) {
test.Errorf("wait returned with unexpected status: %v", err)
}
mutex.Unlock()
wg.Done()
}()
<-doneCh
}
for ii := 0; ii < 10; ii++ {
doneCh := make(chan bool)
wg.Add(1)
go func() {
mutex.Lock()
doneCh <- true
errWait := cond.WaitTimeout(time.After(time.Minute))
if !errs.IsClosed(errWait) {
test.Errorf("timed wait returned with unexpected status: %v", errWait)
}
mutex.Unlock()
wg.Done()
}()
<-doneCh
}
cond.Close()
wg.Wait()
}
func TestBasicController(test *testing.T) {
filePath := "/tmp/test_basic_controller.log"
simpleLog := log.SimpleFileLog{}
if err := simpleLog.Initialize(filePath); err != nil {
test.Fatalf("could not initialize log backend: %v", err)
return
}
logger := simpleLog.NewLogger("test-basic-controller")
logger.Infof("starting new controller test")
controller := &BasicController{}
controller.Initialize(logger)
defer func() {
if err := controller.Close(); err != nil {
test.Errorf("could not close the controller: %v", err)
return
}
lock, errLock := controller.LockAll()
if !errs.IsClosed(errLock) {
test.Errorf("controller issued lock %v after it is closed", lock)
return
}
// Lock and Unlock work even after a Close. Safety is not expected.
foobar := controller.ReadLock("foo", "bar")
foobar.Unlock()
}()
lock1, errLock1 := controller.LockAll()
if errLock1 != nil {
test.Errorf("could not acquire lock1: %v", errLock1)
return
}
lock2, errLock2 := controller.TimedLockAll(time.Millisecond)
if !errs.IsTimeout(errLock2) {
test.Errorf("second lock %v is issued while lock1 %v is active",
lock2, lock1)
return
}
lock1.Unlock()
lock3, errLock3 := controller.TimedLock(time.Millisecond, "a")
if errLock3 != nil {
test.Errorf("could not acquire lock3: %v", errLock3)
return
}
lock4, errLock4 := controller.TimedLock(time.Millisecond, "b")
if errLock4 != nil {
test.Errorf("could not acquire lock4: %v", errLock4)
return
}
lock5, errLock5 := controller.TimedLockAll(time.Millisecond)
if errLock5 == nil {
test.Errorf("lock all lock %v issue while locks %v and %v are active",
lock5, lock3, lock4)
return
}
lock3.Unlock()
lock4.Unlock()
foo := controller.ReadLock("foo")
bar := controller.ReadLock("bar")
bar.Unlock("bar")
foo.Unlock("foo")
baz := controller.ReadLock("baz")
baz.Unlock()
test.Logf("returning")
}
// NotifyAllLearners sends the current vote to all learners.
func (this *Paxos) NotifyAllLearners() (status error) {
if !this.IsConfigured() || !this.IsAcceptor() {
return nil
}
defer func() {
if status != nil && !errs.IsClosed(status) {
now := time.Now()
next := now.Add(this.opts.LearnRetryInterval)
_ = this.alarm.ScheduleAt(this.uid, next, this.NotifyAllLearners)
}
}()
rlock := this.ctlr.ReadLock("acceptor", "config")
// Stop notifications when all learners know the consensus value.
if len(this.doneLearnerSet) == len(this.learnerList) {
rlock.Unlock()
return nil
}
// Make a copy of what we need: learners and the vote map.
numLearners := len(this.learnerList)
learnerList := append([]string{}, this.learnerList...)
votedValueMap := make(map[int64][]byte)
for ballot, value := range this.votedValueMap {
if ackMap := this.learnerAckMap[ballot]; len(ackMap) < numLearners {
votedValueMap[ballot] = value
}
}
rlock.Unlock()
request := thispb.LearnRequest{}
for ballot, value := range votedValueMap {
request.VotedBallotList = append(request.VotedBallotList, ballot)
request.VotedValueList = append(request.VotedValueList, value)
}
message := thispb.PaxosMessage{}
message.LearnRequest = &request
// Send notification to all learners.
reqHeader := this.msn.NewRequest(this.namespace, this.uid,
"ClassicPaxos.Learn", this.opts.LearnTimeout)
defer this.msn.CloseMessage(reqHeader)
count, errSend := msg.SendAllProto(this.msn, learnerList, reqHeader,
&message)
if errSend != nil {
this.Errorf("could not send learn request to all learners: %v", errSend)
return errSend
}
// Wait for responses from all learners.
for ii := 0; ii < count; ii++ {
message := thispb.PaxosMessage{}
resHeader, errRecv := msg.ReceiveProto(this.msn, reqHeader, &message)
if errRecv != nil {
this.Warningf("could not receive learner responses: %v", errRecv)
break
}
learner := resHeader.GetMessengerId()
if message.LearnResponse == nil {
continue
}
response := message.GetLearnResponse()
// Save the learner acknowledgment to the wal.
change := thispb.AcceptorChange{}
change.AckedLearner = proto.String(learner)
if response.GetKnowsChosenValue() {
change.AckedChosenValue = proto.Bool(true)
} else {
for ballot := range votedValueMap {
change.AckedBallotList = append(change.AckedBallotList, ballot)
}
}
if err := this.UpdateAcceptor(&change); err != nil {
this.Errorf("could not update acceptor state: %v", err)
return err
}
}
return nil
}