func TestSimpleHTTPClientDoCancelContextWaitForRoundTrip(t *testing.T) {
tr := newFakeTransport()
c := &simpleHTTPClient{transport: tr}
donechan := make(chan struct{})
ctx, cancel := context.WithCancel(context.Background())
go func() {
c.Do(ctx, &fakeAction{})
close(donechan)
}()
// This should call CancelRequest and begin the cancellation process
cancel()
select {
case <-donechan:
t.Fatalf("simpleHTTPClient.Do should not have exited yet")
default:
}
tr.finishCancel <- struct{}{}
select {
case <-donechan:
//expected behavior
return
case <-time.After(time.Second):
t.Fatalf("simpleHTTPClient.Do did not exit within 1s")
}
}
func TestSimpleHTTPClientDoCancelContextResponseBodyClosed(t *testing.T) {
tr := newFakeTransport()
c := &simpleHTTPClient{transport: tr}
// create an already-cancelled context
ctx, cancel := context.WithCancel(context.Background())
cancel()
body := &checkableReadCloser{ReadCloser: ioutil.NopCloser(strings.NewReader("foo"))}
go func() {
// wait for CancelRequest to be called, informing us that simpleHTTPClient
// knows the context is already timed out
<-tr.startCancel
tr.respchan <- &http.Response{Body: body}
tr.finishCancel <- struct{}{}
}()
_, _, err := c.Do(ctx, &fakeAction{})
if err == nil {
t.Fatalf("expected non-nil error, got nil")
}
if !body.closed {
t.Fatalf("expected closed body")
}
}
// TestMultiNodeStart ensures that a node can be started correctly. The node should
// start with correct configuration change entries, and can accept and commit
// proposals.
func TestMultiNodeStart(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cc := raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1}
ccdata, err := cc.Marshal()
if err != nil {
t.Fatalf("unexpected marshal error: %v", err)
}
wants := []Ready{
{
SoftState: &SoftState{Lead: 1, RaftState: StateLeader},
HardState: raftpb.HardState{Term: 2, Commit: 2, Vote: 1},
Entries: []raftpb.Entry{
{Type: raftpb.EntryConfChange, Term: 1, Index: 1, Data: ccdata},
{Term: 2, Index: 2},
},
CommittedEntries: []raftpb.Entry{
{Type: raftpb.EntryConfChange, Term: 1, Index: 1, Data: ccdata},
{Term: 2, Index: 2},
},
},
{
HardState: raftpb.HardState{Term: 2, Commit: 3, Vote: 1},
Entries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}},
CommittedEntries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}},
},
}
mn := StartMultiNode(1, 10, 1)
storage := NewMemoryStorage()
mn.CreateGroup(1, []Peer{{ID: 1}}, storage)
mn.Campaign(ctx, 1)
gs := <-mn.Ready()
g := gs[1]
if !reflect.DeepEqual(g, wants[0]) {
t.Fatalf("#%d: g = %+v,\n w %+v", 1, g, wants[0])
} else {
storage.Append(g.Entries)
mn.Advance(gs)
}
mn.Propose(ctx, 1, []byte("foo"))
if gs2 := <-mn.Ready(); !reflect.DeepEqual(gs2[1], wants[1]) {
t.Errorf("#%d: g = %+v,\n w %+v", 2, gs2[1], wants[1])
} else {
storage.Append(gs2[1].Entries)
mn.Advance(gs2)
}
select {
case rd := <-mn.Ready():
t.Errorf("unexpected Ready: %+v", rd)
case <-time.After(time.Millisecond):
}
}
// Cancel and Stop should unblock Step()
func TestMultiNodeStepUnblock(t *testing.T) {
// a node without buffer to block step
mn := &multiNode{
propc: make(chan multiMessage),
done: make(chan struct{}),
}
ctx, cancel := context.WithCancel(context.Background())
stopFunc := func() { close(mn.done) }
tests := []struct {
unblock func()
werr error
}{
{stopFunc, ErrStopped},
{cancel, context.Canceled},
}
for i, tt := range tests {
errc := make(chan error, 1)
go func() {
err := mn.Step(ctx, 1, raftpb.Message{Type: raftpb.MsgProp})
errc <- err
}()
tt.unblock()
select {
case err := <-errc:
if err != tt.werr {
t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
}
//clean up side-effect
if ctx.Err() != nil {
ctx = context.TODO()
}
select {
case <-mn.done:
mn.done = make(chan struct{})
default:
}
case <-time.After(time.Millisecond * 100):
t.Errorf("#%d: failed to unblock step", i)
}
}
}
func TestMultiNodeAdvance(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
storage := NewMemoryStorage()
mn := StartMultiNode(1, 10, 1)
mn.CreateGroup(1, []Peer{{ID: 1}}, storage)
mn.Campaign(ctx, 1)
rd1 := <-mn.Ready()
mn.Propose(ctx, 1, []byte("foo"))
select {
case rd2 := <-mn.Ready():
t.Fatalf("unexpected Ready before Advance: %+v", rd2)
case <-time.After(time.Millisecond):
}
storage.Append(rd1[1].Entries)
mn.Advance(rd1)
select {
case <-mn.Ready():
case <-time.After(time.Millisecond):
t.Errorf("expect Ready after Advance, but there is no Ready available")
}
}
func BenchmarkOneNode(b *testing.B) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
n := newNode()
s := NewMemoryStorage()
r := newRaft(1, []uint64{1}, 10, 1, s, 0)
go n.run(r)
defer n.Stop()
n.Campaign(ctx)
for i := 0; i < b.N; i++ {
rd := <-n.Ready()
s.Append(rd.Entries)
n.Advance()
n.Propose(ctx, []byte("foo"))
}
rd := <-n.Ready()
if rd.HardState.Commit != uint64(b.N+1) {
b.Errorf("commit = %d, want %d", rd.HardState.Commit, b.N+1)
}
}
func startPeer(tr http.RoundTripper, urls types.URLs, local, to, cid types.ID, r Raft, fs *stats.FollowerStats, errorc chan error) *peer {
picker := newURLPicker(urls)
p := &peer{
id: to,
r: r,
msgAppWriter: startStreamWriter(to, fs, r),
writer: startStreamWriter(to, fs, r),
pipeline: newPipeline(tr, picker, to, cid, fs, r, errorc),
sendc: make(chan raftpb.Message),
recvc: make(chan raftpb.Message, recvBufSize),
propc: make(chan raftpb.Message, maxPendingProposals),
newURLsC: make(chan types.URLs),
pausec: make(chan struct{}),
resumec: make(chan struct{}),
stopc: make(chan struct{}),
done: make(chan struct{}),
}
// Use go-routine for process of MsgProp because it is
// blocking when there is no leader.
ctx, cancel := context.WithCancel(context.Background())
go func() {
for {
select {
case mm := <-p.propc:
if err := r.Process(ctx, mm); err != nil {
log.Printf("peer: process raft message error: %v", err)
}
case <-p.stopc:
return
}
}
}()
go func() {
var paused bool
msgAppReader := startStreamReader(tr, picker, streamTypeMsgApp, local, to, cid, p.recvc, p.propc)
reader := startStreamReader(tr, picker, streamTypeMessage, local, to, cid, p.recvc, p.propc)
for {
select {
case m := <-p.sendc:
if paused {
continue
}
writec, name := p.pick(m)
select {
case writec <- m:
default:
p.r.ReportUnreachable(m.To)
if isMsgSnap(m) {
p.r.ReportSnapshot(m.To, raft.SnapshotFailure)
}
log.Printf("peer: dropping %s to %s since %s with %d-size buffer is blocked",
m.Type, p.id, name, bufSizeMap[name])
}
case mm := <-p.recvc:
if mm.Type == raftpb.MsgApp {
msgAppReader.updateMsgAppTerm(mm.Term)
}
if err := r.Process(context.TODO(), mm); err != nil {
log.Printf("peer: process raft message error: %v", err)
}
case urls := <-p.newURLsC:
picker.update(urls)
case <-p.pausec:
paused = true
case <-p.resumec:
paused = false
case <-p.stopc:
cancel()
p.msgAppWriter.stop()
p.writer.stop()
p.pipeline.stop()
msgAppReader.stop()
reader.stop()
close(p.done)
return
}
}
}()
return p
}
