func TestSenderPost(t *testing.T) {
tr := &roundTripperRecorder{}
p := NewPeer(tr, "http://10.0.0.1", types.ID(1), types.ID(1), &nopProcessor{}, nil, nil)
if err := p.post([]byte("some data")); err != nil {
t.Fatalf("unexpect post error: %v", err)
}
p.Stop()
if g := tr.Request().Method; g != "POST" {
t.Errorf("method = %s, want %s", g, "POST")
}
if g := tr.Request().URL.String(); g != "http://10.0.0.1" {
t.Errorf("url = %s, want %s", g, "http://10.0.0.1")
}
if g := tr.Request().Header.Get("Content-Type"); g != "application/protobuf" {
t.Errorf("content type = %s, want %s", g, "application/protobuf")
}
if g := tr.Request().Header.Get("X-Etcd-Cluster-ID"); g != "1" {
t.Errorf("cluster id = %s, want %s", g, "1")
}
b, err := ioutil.ReadAll(tr.Request().Body)
if err != nil {
t.Fatalf("unexpected ReadAll error: %v", err)
}
if string(b) != "some data" {
t.Errorf("body = %s, want %s", b, "some data")
}
}
func TestStreamReaderDialRequest(t *testing.T) {
for i, tt := range []streamType{streamTypeMsgApp, streamTypeMessage, streamTypeMsgAppV2} {
tr := &roundTripperRecorder{}
sr := &streamReader{
tr: tr,
picker: mustNewURLPicker(t, []string{"http://localhost:2380"}),
local: types.ID(1),
remote: types.ID(2),
cid: types.ID(1),
msgAppTerm: 1,
}
sr.dial(tt)
req := tr.Request()
wurl := fmt.Sprintf("http://localhost:2380" + tt.endpoint() + "/1")
if req.URL.String() != wurl {
t.Errorf("#%d: url = %s, want %s", i, req.URL.String(), wurl)
}
if w := "GET"; req.Method != w {
t.Errorf("#%d: method = %s, want %s", i, req.Method, w)
}
if g := req.Header.Get("X-Etcd-Cluster-ID"); g != "1" {
t.Errorf("#%d: header X-Etcd-Cluster-ID = %s, want 1", i, g)
}
if g := req.Header.Get("X-Raft-To"); g != "2" {
t.Errorf("#%d: header X-Raft-To = %s, want 2", i, g)
}
if g := req.Header.Get("X-Raft-Term"); tt == streamTypeMsgApp && g != "1" {
t.Errorf("#%d: header X-Raft-Term = %s, want 1", i, g)
}
}
}
func TestTransportAdd(t *testing.T) {
ls := stats.NewLeaderStats("")
term := uint64(10)
tr := &transport{
roundTripper: &roundTripperRecorder{},
leaderStats: ls,
term: term,
peers: make(map[types.ID]Peer),
}
tr.AddPeer(1, []string{"http://localhost:2380"})
if _, ok := ls.Followers["1"]; !ok {
t.Errorf("FollowerStats[1] is nil, want exists")
}
s, ok := tr.peers[types.ID(1)]
if !ok {
tr.Stop()
t.Fatalf("senders[1] is nil, want exists")
}
// duplicate AddPeer is ignored
tr.AddPeer(1, []string{"http://localhost:2380"})
ns := tr.peers[types.ID(1)]
if s != ns {
t.Errorf("sender = %v, want %v", ns, s)
}
tr.Stop()
if g := s.(*peer).msgAppReader.msgAppTerm; g != term {
t.Errorf("peer.term = %d, want %d", g, term)
}
}
func TestPipelinePost(t *testing.T) {
tr := &roundTripperRecorder{}
picker := mustNewURLPicker(t, []string{"http://localhost:2380"})
p := newPipeline(tr, picker, types.ID(2), types.ID(1), types.ID(1), newPeerStatus(types.ID(1)), nil, &fakeRaft{}, nil)
if err := p.post([]byte("some data")); err != nil {
t.Fatalf("unexpect post error: %v", err)
}
p.stop()
if g := tr.Request().Method; g != "POST" {
t.Errorf("method = %s, want %s", g, "POST")
}
if g := tr.Request().URL.String(); g != "http://localhost:2380/raft" {
t.Errorf("url = %s, want %s", g, "http://localhost:2380/raft")
}
if g := tr.Request().Header.Get("Content-Type"); g != "application/protobuf" {
t.Errorf("content type = %s, want %s", g, "application/protobuf")
}
if g := tr.Request().Header.Get("X-Server-Version"); g != version.Version {
t.Errorf("version = %s, want %s", g, version.Version)
}
if g := tr.Request().Header.Get("X-Min-Cluster-Version"); g != version.MinClusterVersion {
t.Errorf("min version = %s, want %s", g, version.MinClusterVersion)
}
if g := tr.Request().Header.Get("X-Etcd-Cluster-ID"); g != "1" {
t.Errorf("cluster id = %s, want %s", g, "1")
}
b, err := ioutil.ReadAll(tr.Request().Body)
if err != nil {
t.Fatalf("unexpected ReadAll error: %v", err)
}
if string(b) != "some data" {
t.Errorf("body = %s, want %s", b, "some data")
}
}
func TestSenderExceedMaximalServing(t *testing.T) {
tr := newRoundTripperBlocker()
fs := &stats.FollowerStats{}
p := NewPeer(tr, "http://10.0.0.1", types.ID(1), types.ID(1), &nopProcessor{}, fs, nil)
// keep the sender busy and make the buffer full
// nothing can go out as we block the sender
for i := 0; i < connPerSender+senderBufSize; i++ {
if err := p.Send(raftpb.Message{}); err != nil {
t.Errorf("send err = %v, want nil", err)
}
// force the sender to grab data
testutil.ForceGosched()
}
// try to send a data when we are sure the buffer is full
if err := p.Send(raftpb.Message{}); err == nil {
t.Errorf("unexpect send success")
}
// unblock the senders and force them to send out the data
tr.unblock()
testutil.ForceGosched()
// It could send new data after previous ones succeed
if err := p.Send(raftpb.Message{}); err != nil {
t.Errorf("send err = %v, want nil", err)
}
p.Stop()
}
func readWAL(waldir string, snap walpb.Snapshot) (w *wal.WAL, id, cid types.ID, st raftpb.HardState, ents []raftpb.Entry) {
var (
err error
wmetadata []byte
)
repaired := false
for {
if w, err = wal.Open(waldir, snap); err != nil {
plog.Fatalf("open wal error: %v", err)
}
if wmetadata, st, ents, err = w.ReadAll(); err != nil {
w.Close()
// we can only repair ErrUnexpectedEOF and we never repair twice.
if repaired || err != io.ErrUnexpectedEOF {
plog.Fatalf("read wal error (%v) and cannot be repaired", err)
}
if !wal.Repair(waldir) {
plog.Fatalf("WAL error (%v) cannot be repaired", err)
} else {
plog.Infof("repaired WAL error (%v)", err)
repaired = true
}
continue
}
break
}
var metadata pb.Metadata
pbutil.MustUnmarshal(&metadata, wmetadata)
id = types.ID(metadata.NodeID)
cid = types.ID(metadata.ClusterID)
return
}
func TestClusterUpdateAttributes(t *testing.T) {
name := "etcd"
clientURLs := []string{"http://127.0.0.1:4001"}
tests := []struct {
mems []*Member
removed map[types.ID]bool
wmems []*Member
}{
// update attributes of existing member
{
[]*Member{
newTestMember(1, nil, "", nil),
},
nil,
[]*Member{
newTestMember(1, nil, name, clientURLs),
},
},
// update attributes of removed member
{
nil,
map[types.ID]bool{types.ID(1): true},
nil,
},
}
for i, tt := range tests {
c := newTestCluster(tt.mems)
c.removed = tt.removed
c.UpdateAttributes(types.ID(1), Attributes{Name: name, ClientURLs: clientURLs})
if g := c.Members(); !reflect.DeepEqual(g, tt.wmems) {
t.Errorf("#%d: members = %+v, want %+v", i, g, tt.wmems)
}
}
}
func (s *EtcdServer) parseProposeCtxErr(err error, start time.Time) error {
switch err {
case context.Canceled:
return ErrCanceled
case context.DeadlineExceeded:
curLeadElected := s.r.leadElectedTime()
prevLeadLost := curLeadElected.Add(-2 * time.Duration(s.cfg.ElectionTicks) * time.Duration(s.cfg.TickMs) * time.Millisecond)
if start.After(prevLeadLost) && start.Before(curLeadElected) {
return ErrTimeoutDueToLeaderFail
}
lead := types.ID(atomic.LoadUint64(&s.r.lead))
switch lead {
case types.ID(raft.None):
// TODO: return error to specify it happens because the cluster does not have leader now
case s.ID():
if !isConnectedToQuorumSince(s.r.transport, start, s.ID(), s.cluster.Members()) {
return ErrTimeoutDueToConnectionLost
}
default:
if !isConnectedSince(s.r.transport, start, lead) {
return ErrTimeoutDueToConnectionLost
}
}
return ErrTimeout
default:
return err
}
}
func (s *EtcdServer) Process(ctx context.Context, m raftpb.Message) error {
if s.cluster.IsIDRemoved(types.ID(m.From)) {
plog.Warningf("reject message from removed member %s", types.ID(m.From).String())
return httptypes.NewHTTPError(http.StatusForbidden, "cannot process message from removed member")
}
if m.Type == raftpb.MsgApp {
s.stats.RecvAppendReq(types.ID(m.From).String(), m.Size())
}
return s.r.Step(ctx, m)
}
func TestServeRaftStreamPrefix(t *testing.T) {
tests := []struct {
path string
wtype streamType
}{
{
RaftStreamPrefix + "/message/1",
streamTypeMessage,
},
{
RaftStreamPrefix + "/msgapp/1",
streamTypeMsgAppV2,
},
// backward compatibility
{
RaftStreamPrefix + "/1",
streamTypeMsgApp,
},
}
for i, tt := range tests {
req, err := http.NewRequest("GET", "http://localhost:2380"+tt.path, nil)
if err != nil {
t.Fatalf("#%d: could not create request: %#v", i, err)
}
req.Header.Set("X-Etcd-Cluster-ID", "1")
req.Header.Set("X-Raft-To", "2")
wterm := "1"
req.Header.Set("X-Raft-Term", wterm)
peer := newFakePeer()
peerGetter := &fakePeerGetter{peers: map[types.ID]Peer{types.ID(1): peer}}
h := newStreamHandler(peerGetter, &fakeRaft{}, types.ID(2), types.ID(1))
rw := httptest.NewRecorder()
go h.ServeHTTP(rw, req)
var conn *outgoingConn
select {
case conn = <-peer.connc:
case <-time.After(time.Second):
t.Fatalf("#%d: failed to attach outgoingConn", i)
}
if g := rw.Header().Get("X-Server-Version"); g != version.Version {
t.Errorf("#%d: X-Server-Version = %s, want %s", i, g, version.Version)
}
if conn.t != tt.wtype {
t.Errorf("#%d: type = %s, want %s", i, conn.t, tt.wtype)
}
if conn.termStr != wterm {
t.Errorf("#%d: term = %s, want %s", i, conn.termStr, wterm)
}
conn.Close()
}
}
func TestTransportRemove(t *testing.T) {
tr := &transport{
leaderStats: stats.NewLeaderStats(""),
peers: make(map[types.ID]*peer),
}
tr.AddPeer(1, []string{"http://a"})
tr.RemovePeer(types.ID(1))
if _, ok := tr.peers[types.ID(1)]; ok {
t.Fatalf("senders[1] exists, want removed")
}
}
func TestTransportUpdate(t *testing.T) {
peer := newFakePeer()
tr := &transport{
peers: map[types.ID]Peer{types.ID(1): peer},
}
u := "http://localhost:2380"
tr.UpdatePeer(types.ID(1), []string{u})
wurls := types.URLs(testutil.MustNewURLs(t, []string{"http://localhost:2380"}))
if !reflect.DeepEqual(peer.urls, wurls) {
t.Errorf("urls = %+v, want %+v", peer.urls, wurls)
}
}
func TestTransportRemove(t *testing.T) {
tr := &transport{
roundTripper: &roundTripperRecorder{},
leaderStats: stats.NewLeaderStats(""),
peers: make(map[types.ID]Peer),
}
tr.AddPeer(1, []string{"http://localhost:2380"})
tr.RemovePeer(types.ID(1))
defer tr.Stop()
if _, ok := tr.peers[types.ID(1)]; ok {
t.Fatalf("senders[1] exists, want removed")
}
}
func GuessNodeID(nodes map[string]uint64, snap4 *Snapshot4, cfg *Config4, name string) uint64 {
var snapNodes map[string]uint64
if snap4 != nil {
snapNodes = snap4.GetNodesFromStore()
}
// First, use the flag, if set.
if name != "" {
log.Printf("Using suggested name %s", name)
if val, ok := nodes[name]; ok {
log.Printf("Assigning %s the ID %s", name, types.ID(val))
return val
}
if snapNodes != nil {
if val, ok := snapNodes[name]; ok {
log.Printf("Assigning %s the ID %s", name, types.ID(val))
return val
}
}
log.Printf("Name not found, autodetecting...")
}
// Next, look at the snapshot peers, if that exists.
if snap4 != nil {
//snapNodes := make(map[string]uint64)
//for _, p := range snap4.Peers {
//m := generateNodeMember(p.Name, p.ConnectionString, "")
//snapNodes[p.Name] = uint64(m.ID)
//}
for _, p := range cfg.Peers {
delete(snapNodes, p.Name)
}
if len(snapNodes) == 1 {
for nodename, id := range nodes {
log.Printf("Autodetected from snapshot: name %s", nodename)
return id
}
}
}
// Then, try and deduce from the log.
for _, p := range cfg.Peers {
delete(nodes, p.Name)
}
if len(nodes) == 1 {
for nodename, id := range nodes {
log.Printf("Autodetected name %s", nodename)
return id
}
}
return 0
}
func readWAL(waldir string, snap walpb.Snapshot) (w *wal.WAL, id, cid types.ID, st raftpb.HardState, ents []raftpb.Entry) {
var err error
if w, err = wal.Open(waldir, snap); err != nil {
log.Fatalf("etcdserver: open wal error: %v", err)
}
var wmetadata []byte
if wmetadata, st, ents, err = w.ReadAll(); err != nil {
log.Fatalf("etcdserver: read wal error: %v", err)
}
var metadata pb.Metadata
pbutil.MustUnmarshal(&metadata, wmetadata)
id = types.ID(metadata.NodeID)
cid = types.ID(metadata.ClusterID)
return
}
// TestPipelineSendFailed tests that when send func meets the post error,
// it increases fail count in stats.
func TestPipelineSendFailed(t *testing.T) {
picker := mustNewURLPicker(t, []string{"http://localhost:2380"})
fs := &stats.FollowerStats{}
p := newPipeline(newRespRoundTripper(0, errors.New("blah")), picker, types.ID(2), types.ID(1), types.ID(1), newPeerStatus(types.ID(1)), fs, &fakeRaft{}, nil)
p.msgc <- raftpb.Message{Type: raftpb.MsgApp}
testutil.WaitSchedule()
p.stop()
fs.Lock()
defer fs.Unlock()
if fs.Counts.Fail != 1 {
t.Errorf("fail = %d, want 1", fs.Counts.Fail)
}
}
func reportSentDuration(sendingType string, m raftpb.Message, duration time.Duration) {
typ := m.Type.String()
if isLinkHeartbeatMessage(m) {
typ = "MsgLinkHeartbeat"
}
msgSentDuration.WithLabelValues(sendingType, types.ID(m.To).String(), typ).Observe(float64(duration.Nanoseconds() / int64(time.Microsecond)))
}
func reportSentFailure(sendingType string, m raftpb.Message) {
typ := m.Type.String()
if isLinkHeartbeatMessage(m) {
typ = "MsgLinkHeartbeat"
}
msgSentFailed.WithLabelValues(sendingType, types.ID(m.To).String(), typ).Inc()
}
func newTestMember(id uint64, peerURLs []string, name string, clientURLs []string) *Member {
return &Member{
ID: types.ID(id),
RaftAttributes: RaftAttributes{PeerURLs: peerURLs},
Attributes: Attributes{Name: name, ClientURLs: clientURLs},
}
}
// TestStreamWriterAttachBadOutgoingConn tests that streamWriter with bad
// outgoingConn will close the outgoingConn and fall back to non-working status.
func TestStreamWriterAttachBadOutgoingConn(t *testing.T) {
sw := startStreamWriter(types.ID(1), newPeerStatus(types.ID(1)), &stats.FollowerStats{}, &fakeRaft{})
defer sw.stop()
wfc := &fakeWriteFlushCloser{err: errors.New("blah")}
sw.attach(&outgoingConn{t: streamTypeMessage, Writer: wfc, Flusher: wfc, Closer: wfc})
sw.msgc <- raftpb.Message{}
testutil.WaitSchedule()
// no longer working
if _, ok := sw.writec(); ok != false {
t.Errorf("working = %v, want false", ok)
}
if wfc.closed != true {
t.Errorf("failed to close the underlying connection")
}
}
func (t *transport) Send(msgs []raftpb.Message) {
for _, m := range msgs {
// intentionally dropped message
if m.To == 0 {
continue
}
to := types.ID(m.To)
if m.Type != raftpb.MsgProp { // proposal message does not have a valid term
t.maybeUpdatePeersTerm(m.Term)
}
p, ok := t.peers[to]
if ok {
if m.Type == raftpb.MsgApp {
t.serverStats.SendAppendReq(m.Size())
}
p.Send(m)
continue
}
g, ok := t.remotes[to]
if ok {
g.Send(m)
continue
}
plog.Debugf("ignored message %s (sent to unknown peer %s)", m.Type, to)
}
}
func (t *transport) Send(msgs []raftpb.Message) {
for _, m := range msgs {
// intentionally dropped message
if m.To == 0 {
continue
}
to := types.ID(m.To)
p, ok := t.peers[to]
if ok {
if m.Type == raftpb.MsgApp {
t.serverStats.SendAppendReq(m.Size())
}
p.Send(m)
continue
}
g, ok := t.remotes[to]
if ok {
g.Send(m)
continue
}
log.Printf("etcdserver: send message to unknown receiver %s", to)
}
}
func (s *EtcdServer) send(ms []raftpb.Message) {
for i, _ := range ms {
if s.cluster.IsIDRemoved(types.ID(ms[i].To)) {
ms[i].To = 0
}
}
s.r.transport.Send(ms)
}
// ValidateConfigurationChange takes a proposed ConfChange and
// ensures that it is still valid.
func (c *cluster) ValidateConfigurationChange(cc raftpb.ConfChange) error {
members, removed := membersFromStore(c.store)
id := types.ID(cc.NodeID)
if removed[id] {
return ErrIDRemoved
}
switch cc.Type {
case raftpb.ConfChangeAddNode:
if members[id] != nil {
return ErrIDExists
}
urls := make(map[string]bool)
for _, m := range members {
for _, u := range m.PeerURLs {
urls[u] = true
}
}
m := new(Member)
if err := json.Unmarshal(cc.Context, m); err != nil {
plog.Panicf("unmarshal member should never fail: %v", err)
}
for _, u := range m.PeerURLs {
if urls[u] {
return ErrPeerURLexists
}
}
case raftpb.ConfChangeRemoveNode:
if members[id] == nil {
return ErrIDNotFound
}
case raftpb.ConfChangeUpdateNode:
if members[id] == nil {
return ErrIDNotFound
}
urls := make(map[string]bool)
for _, m := range members {
if m.ID == id {
continue
}
for _, u := range m.PeerURLs {
urls[u] = true
}
}
m := new(Member)
if err := json.Unmarshal(cc.Context, m); err != nil {
plog.Panicf("unmarshal member should never fail: %v", err)
}
for _, u := range m.PeerURLs {
if urls[u] {
return ErrPeerURLexists
}
}
default:
plog.Panicf("ConfChange type should be either AddNode, RemoveNode or UpdateNode")
}
return nil
}
func (c *cluster) genID() {
mIDs := c.MemberIDs()
b := make([]byte, 8*len(mIDs))
for i, id := range mIDs {
binary.BigEndian.PutUint64(b[8*i:], uint64(id))
}
hash := sha1.Sum(b)
c.id = types.ID(binary.BigEndian.Uint64(hash[:8]))
}
func TestMsgApp(t *testing.T) {
tests := []raftpb.Message{
{
Type: raftpb.MsgApp,
From: 1,
To: 2,
Term: 1,
LogTerm: 1,
Index: 3,
Entries: []raftpb.Entry{{Term: 1, Index: 4}},
},
{
Type: raftpb.MsgApp,
From: 1,
To: 2,
Term: 1,
LogTerm: 1,
Index: 0,
Entries: []raftpb.Entry{
{Term: 1, Index: 1, Data: []byte("some data")},
{Term: 1, Index: 2, Data: []byte("some data")},
{Term: 1, Index: 3, Data: []byte("some data")},
},
},
linkHeartbeatMessage,
}
for i, tt := range tests {
b := &bytes.Buffer{}
enc := &msgAppEncoder{w: b, fs: &stats.FollowerStats{}}
if err := enc.encode(tt); err != nil {
t.Errorf("#%d: unexpected encode message error: %v", i, err)
continue
}
dec := &msgAppDecoder{r: b, local: types.ID(tt.To), remote: types.ID(tt.From), term: tt.Term}
m, err := dec.decode()
if err != nil {
t.Errorf("#%d: unexpected decode message error: %v", i, err)
continue
}
if !reflect.DeepEqual(m, tt) {
t.Errorf("#%d: message = %+v, want %+v", i, m, tt)
}
}
}
// TestSenderSend tests that send func could post data using roundtripper
// and increase success count in stats.
func TestSenderSend(t *testing.T) {
tr := &roundTripperRecorder{}
fs := &stats.FollowerStats{}
p := NewPeer(tr, "http://10.0.0.1", types.ID(1), types.ID(1), &nopProcessor{}, fs, nil)
if err := p.Send(raftpb.Message{Type: raftpb.MsgApp}); err != nil {
t.Fatalf("unexpect send error: %v", err)
}
p.Stop()
if tr.Request() == nil {
t.Errorf("sender fails to post the data")
}
fs.Lock()
defer fs.Unlock()
if fs.Counts.Success != 1 {
t.Errorf("success = %d, want 1", fs.Counts.Success)
}
}
func TestStopBlockedPipeline(t *testing.T) {
picker := mustNewURLPicker(t, []string{"http://localhost:2380"})
p := newPipeline(newRoundTripperBlocker(), picker, types.ID(2), types.ID(1), types.ID(1), newPeerStatus(types.ID(1)), nil, &fakeRaft{}, nil)
// send many messages that most of them will be blocked in buffer
for i := 0; i < connPerPipeline*10; i++ {
p.msgc <- raftpb.Message{}
}
done := make(chan struct{})
go func() {
p.stop()
done <- struct{}{}
}()
select {
case <-done:
case <-time.After(time.Second):
t.Fatalf("failed to stop pipeline in 1s")
}
}
// TestPipelineSend tests that pipeline could send data using roundtripper
// and increase success count in stats.
func TestPipelineSend(t *testing.T) {
tr := &roundTripperRecorder{}
picker := mustNewURLPicker(t, []string{"http://localhost:2380"})
fs := &stats.FollowerStats{}
p := newPipeline(tr, picker, types.ID(2), types.ID(1), types.ID(1), newPeerStatus(types.ID(1)), fs, &fakeRaft{}, nil)
p.msgc <- raftpb.Message{Type: raftpb.MsgApp}
testutil.WaitSchedule()
p.stop()
if tr.Request() == nil {
t.Errorf("sender fails to post the data")
}
fs.Lock()
defer fs.Unlock()
if fs.Counts.Success != 1 {
t.Errorf("success = %d, want 1", fs.Counts.Success)
}
}
// TestStreamWriterAttachOutgoingConn tests that outgoingConn can be attached
// to streamWriter. After that, streamWriter can use it to send messages
// continuously, and closes it when stopped.
func TestStreamWriterAttachOutgoingConn(t *testing.T) {
sw := startStreamWriter(types.ID(1), newPeerStatus(types.ID(1)), &stats.FollowerStats{}, &fakeRaft{})
// the expected initial state of streamWrite is not working
if _, ok := sw.writec(); ok != false {
t.Errorf("initial working status = %v, want false", ok)
}
// repeatitive tests to ensure it can use latest connection
var wfc *fakeWriteFlushCloser
for i := 0; i < 3; i++ {
prevwfc := wfc
wfc = &fakeWriteFlushCloser{}
sw.attach(&outgoingConn{t: streamTypeMessage, Writer: wfc, Flusher: wfc, Closer: wfc})
testutil.WaitSchedule()
// previous attached connection should be closed
if prevwfc != nil && prevwfc.closed != true {
t.Errorf("#%d: close of previous connection = %v, want true", i, prevwfc.closed)
}
// starts working
if _, ok := sw.writec(); ok != true {
t.Errorf("#%d: working status = %v, want true", i, ok)
}
sw.msgc <- raftpb.Message{}
testutil.WaitSchedule()
// still working
if _, ok := sw.writec(); ok != true {
t.Errorf("#%d: working status = %v, want true", i, ok)
}
if wfc.written == 0 {
t.Errorf("#%d: failed to write to the underlying connection", i)
}
}
sw.stop()
// no longer in working status now
if _, ok := sw.writec(); ok != false {
t.Errorf("working status after stop = %v, want false", ok)
}
if wfc.closed != true {
t.Errorf("failed to close the underlying connection")
}
}
