func TestTransportAdd(t *testing.T) {
ls := stats.NewLeaderStats("")
term := uint64(10)
tr := &transport{
roundTripper: &roundTripperRecorder{},
leaderStats: ls,
term: term,
peers: make(map[types.ID]Peer),
}
tr.AddPeer(1, []string{"http://localhost:2380"})
if _, ok := ls.Followers["1"]; !ok {
t.Errorf("FollowerStats[1] is nil, want exists")
}
s, ok := tr.peers[types.ID(1)]
if !ok {
tr.Stop()
t.Fatalf("senders[1] is nil, want exists")
}
// duplicate AddPeer is ignored
tr.AddPeer(1, []string{"http://localhost:2380"})
ns := tr.peers[types.ID(1)]
if s != ns {
t.Errorf("sender = %v, want %v", ns, s)
}
tr.Stop()
if g := s.(*peer).msgAppReader.msgAppTerm; g != term {
t.Errorf("peer.term = %d, want %d", g, term)
}
}
func TestTransportErrorc(t *testing.T) {
errorc := make(chan error, 1)
tr := &transport{
roundTripper: newRespRoundTripper(http.StatusForbidden, nil),
leaderStats: stats.NewLeaderStats(""),
peers: make(map[types.ID]Peer),
errorc: errorc,
}
tr.AddPeer(1, []string{"http://localhost:2380"})
defer tr.Stop()
select {
case <-errorc:
t.Fatalf("received unexpected from errorc")
case <-time.After(10 * time.Millisecond):
}
tr.peers[1].Send(raftpb.Message{})
testutil.WaitSchedule()
select {
case <-errorc:
default:
t.Fatalf("cannot receive error from errorc")
}
}
func TestTransportRemove(t *testing.T) {
tr := &transport{
leaderStats: stats.NewLeaderStats(""),
peers: make(map[types.ID]*peer),
}
tr.AddPeer(1, []string{"http://a"})
tr.RemovePeer(types.ID(1))
if _, ok := tr.peers[types.ID(1)]; ok {
t.Fatalf("senders[1] exists, want removed")
}
}
func TestTransportRemove(t *testing.T) {
tr := &transport{
roundTripper: &roundTripperRecorder{},
leaderStats: stats.NewLeaderStats(""),
peers: make(map[types.ID]Peer),
}
tr.AddPeer(1, []string{"http://localhost:2380"})
tr.RemovePeer(types.ID(1))
defer tr.Stop()
if _, ok := tr.peers[types.ID(1)]; ok {
t.Fatalf("senders[1] exists, want removed")
}
}
func TestTransportAdd(t *testing.T) {
ls := stats.NewLeaderStats("")
tr := &transport{
leaderStats: ls,
peers: make(map[types.ID]*peer),
}
tr.AddPeer(1, []string{"http://a"})
if _, ok := ls.Followers["1"]; !ok {
t.Errorf("FollowerStats[1] is nil, want exists")
}
s, ok := tr.peers[types.ID(1)]
if !ok {
t.Fatalf("senders[1] is nil, want exists")
}
// duplicate AddPeer is ignored
tr.AddPeer(1, []string{"http://a"})
ns := tr.peers[types.ID(1)]
if s != ns {
t.Errorf("sender = %v, want %v", ns, s)
}
}
// TestSendMessageWhenStreamIsBroken tests that message can be sent to the
// remote in a limited time when all underlying connections are broken.
func TestSendMessageWhenStreamIsBroken(t *testing.T) {
// member 1
tr := NewTransporter(&http.Transport{}, types.ID(1), types.ID(1), &fakeRaft{}, nil, newServerStats(), stats.NewLeaderStats("1"))
srv := httptest.NewServer(tr.Handler())
defer srv.Close()
// member 2
recvc := make(chan raftpb.Message, 1)
p := &fakeRaft{recvc: recvc}
tr2 := NewTransporter(&http.Transport{}, types.ID(2), types.ID(1), p, nil, newServerStats(), stats.NewLeaderStats("2"))
srv2 := httptest.NewServer(tr2.Handler())
defer srv2.Close()
tr.AddPeer(types.ID(2), []string{srv2.URL})
defer tr.Stop()
tr2.AddPeer(types.ID(1), []string{srv.URL})
defer tr2.Stop()
if !waitStreamWorking(tr.(*transport).Get(types.ID(2)).(*peer)) {
t.Fatalf("stream from 1 to 2 is not in work as expected")
}
// break the stream
srv.CloseClientConnections()
srv2.CloseClientConnections()
var n int
for {
select {
// TODO: remove this resend logic when we add retry logic into the code
case <-time.After(time.Millisecond):
n++
tr.Send([]raftpb.Message{{Type: raftpb.MsgHeartbeat, From: 1, To: 2, Term: 1, Commit: 3}})
case <-recvc:
if n > 10 {
t.Errorf("disconnection time = %dms, want < 10ms", n)
}
return
}
}
}
func TestSendMessage(t *testing.T) {
// member 1
tr := NewTransporter(&http.Transport{}, types.ID(1), types.ID(1), &fakeRaft{}, nil, newServerStats(), stats.NewLeaderStats("1"))
srv := httptest.NewServer(tr.Handler())
defer srv.Close()
// member 2
recvc := make(chan raftpb.Message, 1)
p := &fakeRaft{recvc: recvc}
tr2 := NewTransporter(&http.Transport{}, types.ID(2), types.ID(1), p, nil, newServerStats(), stats.NewLeaderStats("2"))
srv2 := httptest.NewServer(tr2.Handler())
defer srv2.Close()
tr.AddPeer(types.ID(2), []string{srv2.URL})
defer tr.Stop()
tr2.AddPeer(types.ID(1), []string{srv.URL})
defer tr2.Stop()
if !waitStreamWorking(tr.(*transport).Get(types.ID(2)).(*peer)) {
t.Fatalf("stream from 1 to 2 is not in work as expected")
}
data := []byte("some data")
tests := []raftpb.Message{
// these messages are set to send to itself, which faciliates testing.
{Type: raftpb.MsgProp, From: 1, To: 2, Entries: []raftpb.Entry{{Data: data}}},
// TODO: send out MsgApp which fits msgapp stream but the term doesn't match
{Type: raftpb.MsgApp, From: 1, To: 2, Term: 1, Index: 3, LogTerm: 0, Entries: []raftpb.Entry{{Index: 4, Term: 1, Data: data}}, Commit: 3},
{Type: raftpb.MsgAppResp, From: 1, To: 2, Term: 1, Index: 3},
{Type: raftpb.MsgVote, From: 1, To: 2, Term: 1, Index: 3, LogTerm: 0},
{Type: raftpb.MsgVoteResp, From: 1, To: 2, Term: 1},
{Type: raftpb.MsgSnap, From: 1, To: 2, Term: 1, Snapshot: raftpb.Snapshot{Metadata: raftpb.SnapshotMetadata{Index: 1000, Term: 1}, Data: data}},
{Type: raftpb.MsgHeartbeat, From: 1, To: 2, Term: 1, Commit: 3},
{Type: raftpb.MsgHeartbeatResp, From: 1, To: 2, Term: 1},
}
for i, tt := range tests {
tr.Send([]raftpb.Message{tt})
msg := <-recvc
if !reflect.DeepEqual(msg, tt) {
t.Errorf("#%d: msg = %+v, want %+v", i, msg, tt)
}
}
}
// NewServer creates a new EtcdServer from the supplied configuration. The
// configuration is considered static for the lifetime of the EtcdServer.
func NewServer(cfg *ServerConfig) (*EtcdServer, error) {
st := store.New(StoreClusterPrefix, StoreKeysPrefix)
var w *wal.WAL
var n raft.Node
var s *raft.MemoryStorage
var id types.ID
var cl *cluster
// Run the migrations.
dataVer, err := version.DetectDataDir(cfg.DataDir)
if err != nil {
return nil, err
}
if err := upgradeDataDir(cfg.DataDir, cfg.Name, dataVer); err != nil {
return nil, err
}
haveWAL := wal.Exist(cfg.WALDir())
ss := snap.New(cfg.SnapDir())
var remotes []*Member
switch {
case !haveWAL && !cfg.NewCluster:
if err := cfg.VerifyJoinExisting(); err != nil {
return nil, err
}
cl, err = newClusterFromURLsMap(cfg.InitialClusterToken, cfg.InitialPeerURLsMap)
if err != nil {
return nil, err
}
existingCluster, err := GetClusterFromRemotePeers(getRemotePeerURLs(cl, cfg.Name), cfg.Transport)
if err != nil {
return nil, fmt.Errorf("cannot fetch cluster info from peer urls: %v", err)
}
if err := ValidateClusterAndAssignIDs(cl, existingCluster); err != nil {
return nil, fmt.Errorf("error validating peerURLs %s: %v", existingCluster, err)
}
if !isCompatibleWithCluster(cl, cl.MemberByName(cfg.Name).ID, cfg.Transport) {
return nil, fmt.Errorf("incomptible with current running cluster")
}
remotes = existingCluster.Members()
cl.SetID(existingCluster.id)
cl.SetStore(st)
cfg.Print()
id, n, s, w = startNode(cfg, cl, nil)
case !haveWAL && cfg.NewCluster:
if err := cfg.VerifyBootstrap(); err != nil {
return nil, err
}
cl, err = newClusterFromURLsMap(cfg.InitialClusterToken, cfg.InitialPeerURLsMap)
if err != nil {
return nil, err
}
m := cl.MemberByName(cfg.Name)
if isMemberBootstrapped(cl, cfg.Name, cfg.Transport) {
return nil, fmt.Errorf("member %s has already been bootstrapped", m.ID)
}
if cfg.ShouldDiscover() {
str, err := discovery.JoinCluster(cfg.DiscoveryURL, cfg.DiscoveryProxy, m.ID, cfg.InitialPeerURLsMap.String())
if err != nil {
return nil, err
}
urlsmap, err := types.NewURLsMap(str)
if err != nil {
return nil, err
}
if checkDuplicateURL(urlsmap) {
return nil, fmt.Errorf("discovery cluster %s has duplicate url", urlsmap)
}
if cl, err = newClusterFromURLsMap(cfg.InitialClusterToken, urlsmap); err != nil {
return nil, err
}
}
cl.SetStore(st)
cfg.PrintWithInitial()
id, n, s, w = startNode(cfg, cl, cl.MemberIDs())
case haveWAL:
if err := fileutil.IsDirWriteable(cfg.DataDir); err != nil {
return nil, fmt.Errorf("cannot write to data directory: %v", err)
}
if err := fileutil.IsDirWriteable(cfg.MemberDir()); err != nil {
return nil, fmt.Errorf("cannot write to member directory: %v", err)
}
if cfg.ShouldDiscover() {
plog.Warningf("discovery token ignored since a cluster has already been initialized. Valid log found at %q", cfg.WALDir())
}
snapshot, err := ss.Load()
if err != nil && err != snap.ErrNoSnapshot {
return nil, err
}
if snapshot != nil {
if err := st.Recovery(snapshot.Data); err != nil {
plog.Panicf("recovered store from snapshot error: %v", err)
}
plog.Infof("recovered store from snapshot at index %d", snapshot.Metadata.Index)
}
cfg.Print()
if snapshot != nil {
plog.Infof("loaded cluster information from store: %s", cl)
}
if !cfg.ForceNewCluster {
id, cl, n, s, w = restartNode(cfg, snapshot)
} else {
id, cl, n, s, w = restartAsStandaloneNode(cfg, snapshot)
}
cl.SetStore(st)
cl.Recover()
default:
return nil, fmt.Errorf("unsupported bootstrap config")
}
sstats := &stats.ServerStats{
Name: cfg.Name,
ID: id.String(),
}
sstats.Initialize()
lstats := stats.NewLeaderStats(id.String())
srv := &EtcdServer{
cfg: cfg,
snapCount: cfg.SnapCount,
errorc: make(chan error, 1),
store: st,
r: raftNode{
Node: n,
ticker: time.Tick(time.Duration(cfg.TickMs) * time.Millisecond),
raftStorage: s,
storage: NewStorage(w, ss),
},
id: id,
attributes: Attributes{Name: cfg.Name, ClientURLs: cfg.ClientURLs.StringSlice()},
cluster: cl,
stats: sstats,
lstats: lstats,
SyncTicker: time.Tick(500 * time.Millisecond),
reqIDGen: idutil.NewGenerator(uint8(id), time.Now()),
forceVersionC: make(chan struct{}),
}
// TODO: move transport initialization near the definition of remote
tr := rafthttp.NewTransporter(cfg.Transport, id, cl.ID(), srv, srv.errorc, sstats, lstats)
// add all remotes into transport
for _, m := range remotes {
if m.ID != id {
tr.AddRemote(m.ID, m.PeerURLs)
}
}
for _, m := range cl.Members() {
if m.ID != id {
tr.AddPeer(m.ID, m.PeerURLs)
}
}
srv.r.transport = tr
return srv, nil
}
func BenchmarkSendingMsgApp(b *testing.B) {
// member 1
tr := NewTransporter(&http.Transport{}, types.ID(1), types.ID(1), &fakeRaft{}, nil, newServerStats(), stats.NewLeaderStats("1"))
srv := httptest.NewServer(tr.Handler())
defer srv.Close()
// member 2
r := &countRaft{}
tr2 := NewTransporter(&http.Transport{}, types.ID(2), types.ID(1), r, nil, newServerStats(), stats.NewLeaderStats("2"))
srv2 := httptest.NewServer(tr2.Handler())
defer srv2.Close()
tr.AddPeer(types.ID(2), []string{srv2.URL})
defer tr.Stop()
tr2.AddPeer(types.ID(1), []string{srv.URL})
defer tr2.Stop()
if !waitStreamWorking(tr.(*transport).Get(types.ID(2)).(*peer)) {
b.Fatalf("stream from 1 to 2 is not in work as expected")
}
b.ReportAllocs()
b.SetBytes(64)
b.ResetTimer()
data := make([]byte, 64)
for i := 0; i < b.N; i++ {
tr.Send([]raftpb.Message{
{
Type: raftpb.MsgApp,
From: 1,
To: 2,
Index: uint64(i),
Entries: []raftpb.Entry{
{
Index: uint64(i + 1),
Data: data,
},
},
},
})
}
// wait until all messages are received by the target raft
for r.count() != b.N {
time.Sleep(time.Millisecond)
}
b.StopTimer()
}