func TestSenderPost(t *testing.T) { tr := &roundTripperRecorder{} p := NewPeer(tr, "http://10.0.0.1", types.ID(1), types.ID(1), &nopProcessor{}, nil, nil) if err := p.post([]byte("some data")); err != nil { t.Fatalf("unexpect post error: %v", err) } p.Stop() if g := tr.Request().Method; g != "POST" { t.Errorf("method = %s, want %s", g, "POST") } if g := tr.Request().URL.String(); g != "http://10.0.0.1" { t.Errorf("url = %s, want %s", g, "http://10.0.0.1") } if g := tr.Request().Header.Get("Content-Type"); g != "application/protobuf" { t.Errorf("content type = %s, want %s", g, "application/protobuf") } if g := tr.Request().Header.Get("X-Etcd-Cluster-ID"); g != "1" { t.Errorf("cluster id = %s, want %s", g, "1") } b, err := ioutil.ReadAll(tr.Request().Body) if err != nil { t.Fatalf("unexpected ReadAll error: %v", err) } if string(b) != "some data" { t.Errorf("body = %s, want %s", b, "some data") } }
func TestStreamReaderDialRequest(t *testing.T) { for i, tt := range []streamType{streamTypeMsgApp, streamTypeMessage, streamTypeMsgAppV2} { tr := &roundTripperRecorder{} sr := &streamReader{ tr: tr, picker: mustNewURLPicker(t, []string{"http://localhost:2380"}), local: types.ID(1), remote: types.ID(2), cid: types.ID(1), msgAppTerm: 1, } sr.dial(tt) req := tr.Request() wurl := fmt.Sprintf("http://localhost:2380" + tt.endpoint() + "/1") if req.URL.String() != wurl { t.Errorf("#%d: url = %s, want %s", i, req.URL.String(), wurl) } if w := "GET"; req.Method != w { t.Errorf("#%d: method = %s, want %s", i, req.Method, w) } if g := req.Header.Get("X-Etcd-Cluster-ID"); g != "1" { t.Errorf("#%d: header X-Etcd-Cluster-ID = %s, want 1", i, g) } if g := req.Header.Get("X-Raft-To"); g != "2" { t.Errorf("#%d: header X-Raft-To = %s, want 2", i, g) } if g := req.Header.Get("X-Raft-Term"); tt == streamTypeMsgApp && g != "1" { t.Errorf("#%d: header X-Raft-Term = %s, want 1", i, g) } } }
func TestTransportAdd(t *testing.T) { ls := stats.NewLeaderStats("") term := uint64(10) tr := &transport{ roundTripper: &roundTripperRecorder{}, leaderStats: ls, term: term, peers: make(map[types.ID]Peer), } tr.AddPeer(1, []string{"http://localhost:2380"}) if _, ok := ls.Followers["1"]; !ok { t.Errorf("FollowerStats[1] is nil, want exists") } s, ok := tr.peers[types.ID(1)] if !ok { tr.Stop() t.Fatalf("senders[1] is nil, want exists") } // duplicate AddPeer is ignored tr.AddPeer(1, []string{"http://localhost:2380"}) ns := tr.peers[types.ID(1)] if s != ns { t.Errorf("sender = %v, want %v", ns, s) } tr.Stop() if g := s.(*peer).msgAppReader.msgAppTerm; g != term { t.Errorf("peer.term = %d, want %d", g, term) } }
func TestPipelinePost(t *testing.T) { tr := &roundTripperRecorder{} picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) p := newPipeline(tr, picker, types.ID(2), types.ID(1), types.ID(1), newPeerStatus(types.ID(1)), nil, &fakeRaft{}, nil) if err := p.post([]byte("some data")); err != nil { t.Fatalf("unexpect post error: %v", err) } p.stop() if g := tr.Request().Method; g != "POST" { t.Errorf("method = %s, want %s", g, "POST") } if g := tr.Request().URL.String(); g != "http://localhost:2380/raft" { t.Errorf("url = %s, want %s", g, "http://localhost:2380/raft") } if g := tr.Request().Header.Get("Content-Type"); g != "application/protobuf" { t.Errorf("content type = %s, want %s", g, "application/protobuf") } if g := tr.Request().Header.Get("X-Server-Version"); g != version.Version { t.Errorf("version = %s, want %s", g, version.Version) } if g := tr.Request().Header.Get("X-Min-Cluster-Version"); g != version.MinClusterVersion { t.Errorf("min version = %s, want %s", g, version.MinClusterVersion) } if g := tr.Request().Header.Get("X-Etcd-Cluster-ID"); g != "1" { t.Errorf("cluster id = %s, want %s", g, "1") } b, err := ioutil.ReadAll(tr.Request().Body) if err != nil { t.Fatalf("unexpected ReadAll error: %v", err) } if string(b) != "some data" { t.Errorf("body = %s, want %s", b, "some data") } }
func TestSenderExceedMaximalServing(t *testing.T) { tr := newRoundTripperBlocker() fs := &stats.FollowerStats{} p := NewPeer(tr, "http://10.0.0.1", types.ID(1), types.ID(1), &nopProcessor{}, fs, nil) // keep the sender busy and make the buffer full // nothing can go out as we block the sender for i := 0; i < connPerSender+senderBufSize; i++ { if err := p.Send(raftpb.Message{}); err != nil { t.Errorf("send err = %v, want nil", err) } // force the sender to grab data testutil.ForceGosched() } // try to send a data when we are sure the buffer is full if err := p.Send(raftpb.Message{}); err == nil { t.Errorf("unexpect send success") } // unblock the senders and force them to send out the data tr.unblock() testutil.ForceGosched() // It could send new data after previous ones succeed if err := p.Send(raftpb.Message{}); err != nil { t.Errorf("send err = %v, want nil", err) } p.Stop() }
func readWAL(waldir string, snap walpb.Snapshot) (w *wal.WAL, id, cid types.ID, st raftpb.HardState, ents []raftpb.Entry) { var ( err error wmetadata []byte ) repaired := false for { if w, err = wal.Open(waldir, snap); err != nil { plog.Fatalf("open wal error: %v", err) } if wmetadata, st, ents, err = w.ReadAll(); err != nil { w.Close() // we can only repair ErrUnexpectedEOF and we never repair twice. if repaired || err != io.ErrUnexpectedEOF { plog.Fatalf("read wal error (%v) and cannot be repaired", err) } if !wal.Repair(waldir) { plog.Fatalf("WAL error (%v) cannot be repaired", err) } else { plog.Infof("repaired WAL error (%v)", err) repaired = true } continue } break } var metadata pb.Metadata pbutil.MustUnmarshal(&metadata, wmetadata) id = types.ID(metadata.NodeID) cid = types.ID(metadata.ClusterID) return }
func TestClusterUpdateAttributes(t *testing.T) { name := "etcd" clientURLs := []string{"http://127.0.0.1:4001"} tests := []struct { mems []*Member removed map[types.ID]bool wmems []*Member }{ // update attributes of existing member { []*Member{ newTestMember(1, nil, "", nil), }, nil, []*Member{ newTestMember(1, nil, name, clientURLs), }, }, // update attributes of removed member { nil, map[types.ID]bool{types.ID(1): true}, nil, }, } for i, tt := range tests { c := newTestCluster(tt.mems) c.removed = tt.removed c.UpdateAttributes(types.ID(1), Attributes{Name: name, ClientURLs: clientURLs}) if g := c.Members(); !reflect.DeepEqual(g, tt.wmems) { t.Errorf("#%d: members = %+v, want %+v", i, g, tt.wmems) } } }
func (s *EtcdServer) parseProposeCtxErr(err error, start time.Time) error { switch err { case context.Canceled: return ErrCanceled case context.DeadlineExceeded: curLeadElected := s.r.leadElectedTime() prevLeadLost := curLeadElected.Add(-2 * time.Duration(s.cfg.ElectionTicks) * time.Duration(s.cfg.TickMs) * time.Millisecond) if start.After(prevLeadLost) && start.Before(curLeadElected) { return ErrTimeoutDueToLeaderFail } lead := types.ID(atomic.LoadUint64(&s.r.lead)) switch lead { case types.ID(raft.None): // TODO: return error to specify it happens because the cluster does not have leader now case s.ID(): if !isConnectedToQuorumSince(s.r.transport, start, s.ID(), s.cluster.Members()) { return ErrTimeoutDueToConnectionLost } default: if !isConnectedSince(s.r.transport, start, lead) { return ErrTimeoutDueToConnectionLost } } return ErrTimeout default: return err } }
func (s *EtcdServer) Process(ctx context.Context, m raftpb.Message) error { if s.cluster.IsIDRemoved(types.ID(m.From)) { plog.Warningf("reject message from removed member %s", types.ID(m.From).String()) return httptypes.NewHTTPError(http.StatusForbidden, "cannot process message from removed member") } if m.Type == raftpb.MsgApp { s.stats.RecvAppendReq(types.ID(m.From).String(), m.Size()) } return s.r.Step(ctx, m) }
func TestServeRaftStreamPrefix(t *testing.T) { tests := []struct { path string wtype streamType }{ { RaftStreamPrefix + "/message/1", streamTypeMessage, }, { RaftStreamPrefix + "/msgapp/1", streamTypeMsgAppV2, }, // backward compatibility { RaftStreamPrefix + "/1", streamTypeMsgApp, }, } for i, tt := range tests { req, err := http.NewRequest("GET", "http://localhost:2380"+tt.path, nil) if err != nil { t.Fatalf("#%d: could not create request: %#v", i, err) } req.Header.Set("X-Etcd-Cluster-ID", "1") req.Header.Set("X-Raft-To", "2") wterm := "1" req.Header.Set("X-Raft-Term", wterm) peer := newFakePeer() peerGetter := &fakePeerGetter{peers: map[types.ID]Peer{types.ID(1): peer}} h := newStreamHandler(peerGetter, &fakeRaft{}, types.ID(2), types.ID(1)) rw := httptest.NewRecorder() go h.ServeHTTP(rw, req) var conn *outgoingConn select { case conn = <-peer.connc: case <-time.After(time.Second): t.Fatalf("#%d: failed to attach outgoingConn", i) } if g := rw.Header().Get("X-Server-Version"); g != version.Version { t.Errorf("#%d: X-Server-Version = %s, want %s", i, g, version.Version) } if conn.t != tt.wtype { t.Errorf("#%d: type = %s, want %s", i, conn.t, tt.wtype) } if conn.termStr != wterm { t.Errorf("#%d: term = %s, want %s", i, conn.termStr, wterm) } conn.Close() } }
func TestTransportRemove(t *testing.T) { tr := &transport{ leaderStats: stats.NewLeaderStats(""), peers: make(map[types.ID]*peer), } tr.AddPeer(1, []string{"http://a"}) tr.RemovePeer(types.ID(1)) if _, ok := tr.peers[types.ID(1)]; ok { t.Fatalf("senders[1] exists, want removed") } }
func TestTransportUpdate(t *testing.T) { peer := newFakePeer() tr := &transport{ peers: map[types.ID]Peer{types.ID(1): peer}, } u := "http://localhost:2380" tr.UpdatePeer(types.ID(1), []string{u}) wurls := types.URLs(testutil.MustNewURLs(t, []string{"http://localhost:2380"})) if !reflect.DeepEqual(peer.urls, wurls) { t.Errorf("urls = %+v, want %+v", peer.urls, wurls) } }
func TestTransportRemove(t *testing.T) { tr := &transport{ roundTripper: &roundTripperRecorder{}, leaderStats: stats.NewLeaderStats(""), peers: make(map[types.ID]Peer), } tr.AddPeer(1, []string{"http://localhost:2380"}) tr.RemovePeer(types.ID(1)) defer tr.Stop() if _, ok := tr.peers[types.ID(1)]; ok { t.Fatalf("senders[1] exists, want removed") } }
func GuessNodeID(nodes map[string]uint64, snap4 *Snapshot4, cfg *Config4, name string) uint64 { var snapNodes map[string]uint64 if snap4 != nil { snapNodes = snap4.GetNodesFromStore() } // First, use the flag, if set. if name != "" { log.Printf("Using suggested name %s", name) if val, ok := nodes[name]; ok { log.Printf("Assigning %s the ID %s", name, types.ID(val)) return val } if snapNodes != nil { if val, ok := snapNodes[name]; ok { log.Printf("Assigning %s the ID %s", name, types.ID(val)) return val } } log.Printf("Name not found, autodetecting...") } // Next, look at the snapshot peers, if that exists. if snap4 != nil { //snapNodes := make(map[string]uint64) //for _, p := range snap4.Peers { //m := generateNodeMember(p.Name, p.ConnectionString, "") //snapNodes[p.Name] = uint64(m.ID) //} for _, p := range cfg.Peers { delete(snapNodes, p.Name) } if len(snapNodes) == 1 { for nodename, id := range nodes { log.Printf("Autodetected from snapshot: name %s", nodename) return id } } } // Then, try and deduce from the log. for _, p := range cfg.Peers { delete(nodes, p.Name) } if len(nodes) == 1 { for nodename, id := range nodes { log.Printf("Autodetected name %s", nodename) return id } } return 0 }
func readWAL(waldir string, snap walpb.Snapshot) (w *wal.WAL, id, cid types.ID, st raftpb.HardState, ents []raftpb.Entry) { var err error if w, err = wal.Open(waldir, snap); err != nil { log.Fatalf("etcdserver: open wal error: %v", err) } var wmetadata []byte if wmetadata, st, ents, err = w.ReadAll(); err != nil { log.Fatalf("etcdserver: read wal error: %v", err) } var metadata pb.Metadata pbutil.MustUnmarshal(&metadata, wmetadata) id = types.ID(metadata.NodeID) cid = types.ID(metadata.ClusterID) return }
// TestPipelineSendFailed tests that when send func meets the post error, // it increases fail count in stats. func TestPipelineSendFailed(t *testing.T) { picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) fs := &stats.FollowerStats{} p := newPipeline(newRespRoundTripper(0, errors.New("blah")), picker, types.ID(2), types.ID(1), types.ID(1), newPeerStatus(types.ID(1)), fs, &fakeRaft{}, nil) p.msgc <- raftpb.Message{Type: raftpb.MsgApp} testutil.WaitSchedule() p.stop() fs.Lock() defer fs.Unlock() if fs.Counts.Fail != 1 { t.Errorf("fail = %d, want 1", fs.Counts.Fail) } }
func reportSentDuration(sendingType string, m raftpb.Message, duration time.Duration) { typ := m.Type.String() if isLinkHeartbeatMessage(m) { typ = "MsgLinkHeartbeat" } msgSentDuration.WithLabelValues(sendingType, types.ID(m.To).String(), typ).Observe(float64(duration.Nanoseconds() / int64(time.Microsecond))) }
func reportSentFailure(sendingType string, m raftpb.Message) { typ := m.Type.String() if isLinkHeartbeatMessage(m) { typ = "MsgLinkHeartbeat" } msgSentFailed.WithLabelValues(sendingType, types.ID(m.To).String(), typ).Inc() }
func newTestMember(id uint64, peerURLs []string, name string, clientURLs []string) *Member { return &Member{ ID: types.ID(id), RaftAttributes: RaftAttributes{PeerURLs: peerURLs}, Attributes: Attributes{Name: name, ClientURLs: clientURLs}, } }
// TestStreamWriterAttachBadOutgoingConn tests that streamWriter with bad // outgoingConn will close the outgoingConn and fall back to non-working status. func TestStreamWriterAttachBadOutgoingConn(t *testing.T) { sw := startStreamWriter(types.ID(1), newPeerStatus(types.ID(1)), &stats.FollowerStats{}, &fakeRaft{}) defer sw.stop() wfc := &fakeWriteFlushCloser{err: errors.New("blah")} sw.attach(&outgoingConn{t: streamTypeMessage, Writer: wfc, Flusher: wfc, Closer: wfc}) sw.msgc <- raftpb.Message{} testutil.WaitSchedule() // no longer working if _, ok := sw.writec(); ok != false { t.Errorf("working = %v, want false", ok) } if wfc.closed != true { t.Errorf("failed to close the underlying connection") } }
func (t *transport) Send(msgs []raftpb.Message) { for _, m := range msgs { // intentionally dropped message if m.To == 0 { continue } to := types.ID(m.To) if m.Type != raftpb.MsgProp { // proposal message does not have a valid term t.maybeUpdatePeersTerm(m.Term) } p, ok := t.peers[to] if ok { if m.Type == raftpb.MsgApp { t.serverStats.SendAppendReq(m.Size()) } p.Send(m) continue } g, ok := t.remotes[to] if ok { g.Send(m) continue } plog.Debugf("ignored message %s (sent to unknown peer %s)", m.Type, to) } }
func (t *transport) Send(msgs []raftpb.Message) { for _, m := range msgs { // intentionally dropped message if m.To == 0 { continue } to := types.ID(m.To) p, ok := t.peers[to] if ok { if m.Type == raftpb.MsgApp { t.serverStats.SendAppendReq(m.Size()) } p.Send(m) continue } g, ok := t.remotes[to] if ok { g.Send(m) continue } log.Printf("etcdserver: send message to unknown receiver %s", to) } }
func (s *EtcdServer) send(ms []raftpb.Message) { for i, _ := range ms { if s.cluster.IsIDRemoved(types.ID(ms[i].To)) { ms[i].To = 0 } } s.r.transport.Send(ms) }
// ValidateConfigurationChange takes a proposed ConfChange and // ensures that it is still valid. func (c *cluster) ValidateConfigurationChange(cc raftpb.ConfChange) error { members, removed := membersFromStore(c.store) id := types.ID(cc.NodeID) if removed[id] { return ErrIDRemoved } switch cc.Type { case raftpb.ConfChangeAddNode: if members[id] != nil { return ErrIDExists } urls := make(map[string]bool) for _, m := range members { for _, u := range m.PeerURLs { urls[u] = true } } m := new(Member) if err := json.Unmarshal(cc.Context, m); err != nil { plog.Panicf("unmarshal member should never fail: %v", err) } for _, u := range m.PeerURLs { if urls[u] { return ErrPeerURLexists } } case raftpb.ConfChangeRemoveNode: if members[id] == nil { return ErrIDNotFound } case raftpb.ConfChangeUpdateNode: if members[id] == nil { return ErrIDNotFound } urls := make(map[string]bool) for _, m := range members { if m.ID == id { continue } for _, u := range m.PeerURLs { urls[u] = true } } m := new(Member) if err := json.Unmarshal(cc.Context, m); err != nil { plog.Panicf("unmarshal member should never fail: %v", err) } for _, u := range m.PeerURLs { if urls[u] { return ErrPeerURLexists } } default: plog.Panicf("ConfChange type should be either AddNode, RemoveNode or UpdateNode") } return nil }
func (c *cluster) genID() { mIDs := c.MemberIDs() b := make([]byte, 8*len(mIDs)) for i, id := range mIDs { binary.BigEndian.PutUint64(b[8*i:], uint64(id)) } hash := sha1.Sum(b) c.id = types.ID(binary.BigEndian.Uint64(hash[:8])) }
func TestMsgApp(t *testing.T) { tests := []raftpb.Message{ { Type: raftpb.MsgApp, From: 1, To: 2, Term: 1, LogTerm: 1, Index: 3, Entries: []raftpb.Entry{{Term: 1, Index: 4}}, }, { Type: raftpb.MsgApp, From: 1, To: 2, Term: 1, LogTerm: 1, Index: 0, Entries: []raftpb.Entry{ {Term: 1, Index: 1, Data: []byte("some data")}, {Term: 1, Index: 2, Data: []byte("some data")}, {Term: 1, Index: 3, Data: []byte("some data")}, }, }, linkHeartbeatMessage, } for i, tt := range tests { b := &bytes.Buffer{} enc := &msgAppEncoder{w: b, fs: &stats.FollowerStats{}} if err := enc.encode(tt); err != nil { t.Errorf("#%d: unexpected encode message error: %v", i, err) continue } dec := &msgAppDecoder{r: b, local: types.ID(tt.To), remote: types.ID(tt.From), term: tt.Term} m, err := dec.decode() if err != nil { t.Errorf("#%d: unexpected decode message error: %v", i, err) continue } if !reflect.DeepEqual(m, tt) { t.Errorf("#%d: message = %+v, want %+v", i, m, tt) } } }
// TestSenderSend tests that send func could post data using roundtripper // and increase success count in stats. func TestSenderSend(t *testing.T) { tr := &roundTripperRecorder{} fs := &stats.FollowerStats{} p := NewPeer(tr, "http://10.0.0.1", types.ID(1), types.ID(1), &nopProcessor{}, fs, nil) if err := p.Send(raftpb.Message{Type: raftpb.MsgApp}); err != nil { t.Fatalf("unexpect send error: %v", err) } p.Stop() if tr.Request() == nil { t.Errorf("sender fails to post the data") } fs.Lock() defer fs.Unlock() if fs.Counts.Success != 1 { t.Errorf("success = %d, want 1", fs.Counts.Success) } }
func TestStopBlockedPipeline(t *testing.T) { picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) p := newPipeline(newRoundTripperBlocker(), picker, types.ID(2), types.ID(1), types.ID(1), newPeerStatus(types.ID(1)), nil, &fakeRaft{}, nil) // send many messages that most of them will be blocked in buffer for i := 0; i < connPerPipeline*10; i++ { p.msgc <- raftpb.Message{} } done := make(chan struct{}) go func() { p.stop() done <- struct{}{} }() select { case <-done: case <-time.After(time.Second): t.Fatalf("failed to stop pipeline in 1s") } }
// TestPipelineSend tests that pipeline could send data using roundtripper // and increase success count in stats. func TestPipelineSend(t *testing.T) { tr := &roundTripperRecorder{} picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) fs := &stats.FollowerStats{} p := newPipeline(tr, picker, types.ID(2), types.ID(1), types.ID(1), newPeerStatus(types.ID(1)), fs, &fakeRaft{}, nil) p.msgc <- raftpb.Message{Type: raftpb.MsgApp} testutil.WaitSchedule() p.stop() if tr.Request() == nil { t.Errorf("sender fails to post the data") } fs.Lock() defer fs.Unlock() if fs.Counts.Success != 1 { t.Errorf("success = %d, want 1", fs.Counts.Success) } }
// TestStreamWriterAttachOutgoingConn tests that outgoingConn can be attached // to streamWriter. After that, streamWriter can use it to send messages // continuously, and closes it when stopped. func TestStreamWriterAttachOutgoingConn(t *testing.T) { sw := startStreamWriter(types.ID(1), newPeerStatus(types.ID(1)), &stats.FollowerStats{}, &fakeRaft{}) // the expected initial state of streamWrite is not working if _, ok := sw.writec(); ok != false { t.Errorf("initial working status = %v, want false", ok) } // repeatitive tests to ensure it can use latest connection var wfc *fakeWriteFlushCloser for i := 0; i < 3; i++ { prevwfc := wfc wfc = &fakeWriteFlushCloser{} sw.attach(&outgoingConn{t: streamTypeMessage, Writer: wfc, Flusher: wfc, Closer: wfc}) testutil.WaitSchedule() // previous attached connection should be closed if prevwfc != nil && prevwfc.closed != true { t.Errorf("#%d: close of previous connection = %v, want true", i, prevwfc.closed) } // starts working if _, ok := sw.writec(); ok != true { t.Errorf("#%d: working status = %v, want true", i, ok) } sw.msgc <- raftpb.Message{} testutil.WaitSchedule() // still working if _, ok := sw.writec(); ok != true { t.Errorf("#%d: working status = %v, want true", i, ok) } if wfc.written == 0 { t.Errorf("#%d: failed to write to the underlying connection", i) } } sw.stop() // no longer in working status now if _, ok := sw.writec(); ok != false { t.Errorf("working status after stop = %v, want false", ok) } if wfc.closed != true { t.Errorf("failed to close the underlying connection") } }