func TestSenderExceedMaximalServing(t *testing.T) {
tr := newRoundTripperBlocker()
fs := &stats.FollowerStats{}
s := NewSender(tr, "http://10.0.0.1", types.ID(1), &nopProcessor{}, fs, nil)
// keep the sender busy and make the buffer full
// nothing can go out as we block the sender
for i := 0; i < connPerSender+senderBufSize; i++ {
if err := s.Send(raftpb.Message{}); err != nil {
t.Errorf("send err = %v, want nil", err)
}
// force the sender to grab data
testutil.ForceGosched()
}
// try to send a data when we are sure the buffer is full
if err := s.Send(raftpb.Message{}); err == nil {
t.Errorf("unexpect send success")
}
// unblock the senders and force them to send out the data
tr.unblock()
testutil.ForceGosched()
// It could send new data after previous ones succeed
if err := s.Send(raftpb.Message{}); err != nil {
t.Errorf("send err = %v, want nil", err)
}
s.Stop()
}
// TestBlockProposal ensures that node will block proposal when it does not
// know who is the current leader; node will accept proposal when it knows
// who is the current leader.
func TestBlockProposal(t *testing.T) {
n := newNode()
r := newRaft(1, []uint64{1}, 10, 1)
go n.run(r)
defer n.Stop()
errc := make(chan error, 1)
go func() {
errc <- n.Propose(context.TODO(), []byte("somedata"))
}()
testutil.ForceGosched()
select {
case err := <-errc:
t.Errorf("err = %v, want blocking", err)
default:
}
n.Campaign(context.TODO())
testutil.ForceGosched()
select {
case err := <-errc:
if err != nil {
t.Errorf("err = %v, want %v", err, nil)
}
default:
t.Errorf("blocking proposal, want unblocking")
}
}
func TestTransportErrorc(t *testing.T) {
errorc := make(chan error, 1)
tr := &transport{
roundTripper: newRespRoundTripper(http.StatusForbidden, nil),
leaderStats: stats.NewLeaderStats(""),
peers: make(map[types.ID]Peer),
errorc: errorc,
}
tr.AddPeer(1, []string{"http://localhost:7001"})
defer tr.Stop()
select {
case <-errorc:
t.Fatalf("received unexpected from errorc")
case <-time.After(10 * time.Millisecond):
}
tr.peers[1].Send(raftpb.Message{})
testutil.ForceGosched()
select {
case <-errorc:
default:
t.Fatalf("cannot receive error from errorc")
}
}
// TestCompacts ensures Node.Compact creates a correct raft snapshot and compacts
// the raft log (call raft.compact)
func TestNodeCompact(t *testing.T) {
ctx := context.Background()
n := newNode()
r := newRaft(1, []uint64{1}, 10, 1)
go n.run(r)
n.Campaign(ctx)
n.Propose(ctx, []byte("foo"))
w := raftpb.Snapshot{
Term: 1,
Index: 2, // one nop + one proposal
Data: []byte("a snapshot"),
Nodes: []uint64{1},
}
testutil.ForceGosched()
select {
case <-n.Ready():
default:
t.Fatalf("unexpected proposal failure: unable to commit entry")
}
n.Compact(w.Index, w.Nodes, w.Data)
testutil.ForceGosched()
select {
case rd := <-n.Ready():
if !reflect.DeepEqual(rd.Snapshot, w) {
t.Errorf("snap = %+v, want %+v", rd.Snapshot, w)
}
default:
t.Fatalf("unexpected compact failure: unable to create a snapshot")
}
testutil.ForceGosched()
// TODO: this test the run updates the snapi correctly... should be tested
// separately with other kinds of updates
select {
case <-n.Ready():
t.Fatalf("unexpected more ready")
default:
}
n.Stop()
if r.raftLog.offset != w.Index {
t.Errorf("log.offset = %d, want %d", r.raftLog.offset, w.Index)
}
}
// TestStreamWriterAttachOutgoingConn tests that outgoingConn can be attached
// to streamWriter. After that, streamWriter can use it to send messages
// continuously, and closes it when stopped.
func TestStreamWriterAttachOutgoingConn(t *testing.T) {
sw := startStreamWriter(types.ID(1), &stats.FollowerStats{}, &fakeRaft{})
// the expected initial state of streamWrite is not working
if _, ok := sw.writec(); ok != false {
t.Errorf("initial working status = %v, want false", ok)
}
// repeatitive tests to ensure it can use latest connection
var wfc *fakeWriteFlushCloser
for i := 0; i < 3; i++ {
prevwfc := wfc
wfc = &fakeWriteFlushCloser{}
sw.attach(&outgoingConn{t: streamTypeMessage, Writer: wfc, Flusher: wfc, Closer: wfc})
testutil.ForceGosched()
// previous attached connection should be closed
if prevwfc != nil && prevwfc.closed != true {
t.Errorf("#%d: close of previous connection = %v, want true", i, prevwfc.closed)
}
// starts working
if _, ok := sw.writec(); ok != true {
t.Errorf("#%d: working status = %v, want true", i, ok)
}
sw.msgc <- raftpb.Message{}
testutil.ForceGosched()
// still working
if _, ok := sw.writec(); ok != true {
t.Errorf("#%d: working status = %v, want true", i, ok)
}
if wfc.written == 0 {
t.Errorf("#%d: failed to write to the underlying connection", i)
}
}
sw.stop()
// no longer in working status now
if _, ok := sw.writec(); ok != false {
t.Errorf("working status after stop = %v, want false", ok)
}
if wfc.closed != true {
t.Errorf("failed to close the underlying connection")
}
}
func TestPipelineExceedMaximalServing(t *testing.T) {
tr := newRoundTripperBlocker()
picker := mustNewURLPicker(t, []string{"http://localhost:7001"})
fs := &stats.FollowerStats{}
p := newPipeline(tr, picker, types.ID(1), types.ID(1), fs, &fakeRaft{}, nil)
// keep the sender busy and make the buffer full
// nothing can go out as we block the sender
testutil.ForceGosched()
for i := 0; i < connPerPipeline+pipelineBufSize; i++ {
select {
case p.msgc <- raftpb.Message{}:
default:
t.Errorf("failed to send out message")
}
// force the sender to grab data
testutil.ForceGosched()
}
// try to send a data when we are sure the buffer is full
select {
case p.msgc <- raftpb.Message{}:
t.Errorf("unexpected message sendout")
default:
}
// unblock the senders and force them to send out the data
tr.unblock()
testutil.ForceGosched()
// It could send new data after previous ones succeed
select {
case p.msgc <- raftpb.Message{}:
default:
t.Errorf("failed to send out message")
}
p.stop()
}
// TestStreamWriterAttachBadOutgoingConn tests that streamWriter with bad
// outgoingConn will close the outgoingConn and fall back to non-working status.
func TestStreamWriterAttachBadOutgoingConn(t *testing.T) {
sw := startStreamWriter(types.ID(1), &stats.FollowerStats{}, &fakeRaft{})
defer sw.stop()
wfc := &fakeWriteFlushCloser{err: errors.New("blah")}
sw.attach(&outgoingConn{t: streamTypeMessage, Writer: wfc, Flusher: wfc, Closer: wfc})
sw.msgc <- raftpb.Message{}
testutil.ForceGosched()
// no longer working
if _, ok := sw.writec(); ok != false {
t.Errorf("working = %v, want false", ok)
}
if wfc.closed != true {
t.Errorf("failed to close the underlying connection")
}
}
func TestSendHubShouldStop(t *testing.T) {
tr := newRespRoundTripper(http.StatusForbidden, nil)
cl := newTestCluster(nil)
ls := stats.NewLeaderStats("")
h := newSendHub(tr, cl, nil, nil, ls)
m := newTestMember(1, []string{"http://a"}, "", nil)
h.Add(m)
shouldstop := h.ShouldStopNotify()
select {
case <-shouldstop:
t.Fatalf("received unexpected shouldstop notification")
case <-time.After(10 * time.Millisecond):
}
h.senders[1].Send(raftpb.Message{})
testutil.ForceGosched()
select {
case <-shouldstop:
default:
t.Fatalf("cannot receive stop notification")
}
}