// TestBlockProposal ensures that node will block proposal when it does not
// know who is the current leader; node will accept proposal when it knows
// who is the current leader.
func TestBlockProposal(t *testing.T) {
n := newNode()
r := newRaft(1, []uint64{1}, 10, 1, NewMemoryStorage(), 0)
go n.run(r)
defer n.Stop()
errc := make(chan error, 1)
go func() {
errc <- n.Propose(context.TODO(), []byte("somedata"))
}()
testutil.ForceGosched()
select {
case err := <-errc:
t.Errorf("err = %v, want blocking", err)
default:
}
n.Campaign(context.TODO())
testutil.ForceGosched()
select {
case err := <-errc:
if err != nil {
t.Errorf("err = %v, want %v", err, nil)
}
default:
t.Errorf("blocking proposal, want unblocking")
}
}
// TestNodePropose ensures that node.Propose sends the given proposal to the underlying raft.
func TestNodePropose(t *testing.T) {
msgs := []raftpb.Message{}
appendStep := func(r *raft, m raftpb.Message) {
msgs = append(msgs, m)
}
n := newNode()
s := NewMemoryStorage()
r := newRaft(1, []uint64{1}, 10, 1, s, 0)
go n.run(r)
n.Campaign(context.TODO())
for {
rd := <-n.Ready()
s.Append(rd.Entries)
// change the step function to appendStep until this raft becomes leader
if rd.SoftState.Lead == r.id {
r.step = appendStep
n.Advance()
break
}
n.Advance()
}
n.Propose(context.TODO(), []byte("somedata"))
n.Stop()
if len(msgs) != 1 {
t.Fatalf("len(msgs) = %d, want %d", len(msgs), 1)
}
if msgs[0].Type != raftpb.MsgProp {
t.Errorf("msg type = %d, want %d", msgs[0].Type, raftpb.MsgProp)
}
if !reflect.DeepEqual(msgs[0].Entries[0].Data, []byte("somedata")) {
t.Errorf("data = %v, want %v", msgs[0].Entries[0].Data, []byte("somedata"))
}
}
// TestMultiNodeProposeConfig ensures that multiNode.ProposeConfChange
// sends the given configuration proposal to the underlying raft.
func TestMultiNodeProposeConfig(t *testing.T) {
mn := newMultiNode(1)
go mn.run()
s := NewMemoryStorage()
mn.CreateGroup(1, newTestConfig(1, nil, 10, 1, s), []Peer{{ID: 1}})
mn.Campaign(context.TODO(), 1)
proposed := false
var lastIndex uint64
var ccdata []byte
for {
rds := <-mn.Ready()
rd := rds[1]
s.Append(rd.Entries)
// change the step function to appendStep until this raft becomes leader
if !proposed && rd.SoftState.Lead == mn.id {
cc := raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1}
var err error
ccdata, err = cc.Marshal()
if err != nil {
t.Fatal(err)
}
mn.ProposeConfChange(context.TODO(), 1, cc)
proposed = true
}
mn.Advance(rds)
var err error
lastIndex, err = s.LastIndex()
if err != nil {
t.Fatal(err)
}
if lastIndex >= 3 {
break
}
}
mn.Stop()
entries, err := s.Entries(lastIndex, lastIndex+1, noLimit)
if err != nil {
t.Fatal(err)
}
if len(entries) != 1 {
t.Fatalf("len(entries) = %d, want %d", len(entries), 1)
}
if entries[0].Type != raftpb.EntryConfChange {
t.Fatalf("type = %v, want %v", entries[0].Type, raftpb.EntryConfChange)
}
if !bytes.Equal(entries[0].Data, ccdata) {
t.Errorf("data = %v, want %v", entries[0].Data, ccdata)
}
}
func TestBasicProgress(t *testing.T) {
peers := []raft.Peer{{1, nil}, {2, nil}, {3, nil}, {4, nil}, {5, nil}}
nt := newRaftNetwork(1, 2, 3, 4, 5)
nodes := make([]*node, 0)
for i := 1; i <= 5; i++ {
n := startNode(uint64(i), peers, nt.nodeNetwork(uint64(i)))
nodes = append(nodes, n)
}
time.Sleep(10 * time.Millisecond)
for i := 0; i < 10000; i++ {
nodes[0].Propose(context.TODO(), []byte("somedata"))
}
time.Sleep(500 * time.Millisecond)
for _, n := range nodes {
n.stop()
if n.state.Commit != 10006 {
t.Errorf("commit = %d, want = 10006", n.state.Commit)
}
}
}
// TestMultiNodeStep ensures that multiNode.Step sends MsgProp to propc
// chan and other kinds of messages to recvc chan.
func TestMultiNodeStep(t *testing.T) {
for i, msgn := range raftpb.MessageType_name {
mn := &multiNode{
propc: make(chan multiMessage, 1),
recvc: make(chan multiMessage, 1),
}
msgt := raftpb.MessageType(i)
mn.Step(context.TODO(), 1, raftpb.Message{Type: msgt})
// Proposal goes to proc chan. Others go to recvc chan.
if msgt == raftpb.MsgProp {
select {
case <-mn.propc:
default:
t.Errorf("%d: cannot receive %s on propc chan", msgt, msgn)
}
} else {
if msgt == raftpb.MsgBeat || msgt == raftpb.MsgHup || msgt == raftpb.MsgUnreachable || msgt == raftpb.MsgSnapStatus {
select {
case <-mn.recvc:
t.Errorf("%d: step should ignore %s", msgt, msgn)
default:
}
} else {
select {
case <-mn.recvc:
default:
t.Errorf("%d: cannot receive %s on recvc chan", msgt, msgn)
}
}
}
}
}
func BenchmarkProposal3Nodes(b *testing.B) {
peers := []raft.Peer{{1, nil}, {2, nil}, {3, nil}}
nt := newRaftNetwork(1, 2, 3)
nodes := make([]*node, 0)
for i := 1; i <= 3; i++ {
n := startNode(uint64(i), peers, nt.nodeNetwork(uint64(i)))
nodes = append(nodes, n)
}
// get ready and warm up
time.Sleep(50 * time.Millisecond)
b.ResetTimer()
for i := 0; i < b.N; i++ {
nodes[0].Propose(context.TODO(), []byte("somedata"))
}
for _, n := range nodes {
if n.state.Commit != uint64(b.N+4) {
continue
}
}
b.StopTimer()
for _, n := range nodes {
n.stop()
}
}
func (n *node) start() {
n.stopc = make(chan struct{})
ticker := time.Tick(5 * time.Millisecond)
go func() {
for {
select {
case <-ticker:
n.Tick()
case rd := <-n.Ready():
if !raft.IsEmptyHardState(rd.HardState) {
n.state = rd.HardState
n.storage.SetHardState(n.state)
}
n.storage.Append(rd.Entries)
time.Sleep(time.Millisecond)
// TODO: make send async, more like real world...
for _, m := range rd.Messages {
n.iface.send(m)
}
n.Advance()
case m := <-n.iface.recv():
n.Step(context.TODO(), m)
case <-n.stopc:
n.Stop()
log.Printf("raft.%d: stop", n.id)
n.Node = nil
close(n.stopc)
return
case p := <-n.pausec:
recvms := make([]raftpb.Message, 0)
for p {
select {
case m := <-n.iface.recv():
recvms = append(recvms, m)
case p = <-n.pausec:
}
}
// step all pending messages
for _, m := range recvms {
n.Step(context.TODO(), m)
}
}
}
}()
}
// TestMultiNodePropose ensures that node.Propose sends the given proposal to the underlying raft.
func TestMultiNodePropose(t *testing.T) {
mn := newMultiNode(1)
go mn.run()
s := NewMemoryStorage()
mn.CreateGroup(1, newTestConfig(1, nil, 10, 1, s), []Peer{{ID: 1}})
mn.Campaign(context.TODO(), 1)
proposed := false
for {
rds := <-mn.Ready()
rd := rds[1]
s.Append(rd.Entries)
// Once we are the leader, propose a command.
if !proposed && rd.SoftState.Lead == mn.id {
mn.Propose(context.TODO(), 1, []byte("somedata"))
proposed = true
}
mn.Advance(rds)
// Exit when we have three entries: one ConfChange, one no-op for the election,
// and our proposed command.
lastIndex, err := s.LastIndex()
if err != nil {
t.Fatal(err)
}
if lastIndex >= 3 {
break
}
}
mn.Stop()
lastIndex, err := s.LastIndex()
if err != nil {
t.Fatal(err)
}
entries, err := s.Entries(lastIndex, lastIndex+1, noLimit)
if err != nil {
t.Fatal(err)
}
if len(entries) != 1 {
t.Fatalf("len(entries) = %d, want %d", len(entries), 1)
}
if !bytes.Equal(entries[0].Data, []byte("somedata")) {
t.Errorf("entries[0].Data = %v, want %v", entries[0].Data, []byte("somedata"))
}
}
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
w.Header().Set("Allow", "POST")
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
return
}
if err := checkVersionCompability(r.Header.Get("X-Server-From"), serverVersion(r.Header), minClusterVersion(r.Header)); err != nil {
plog.Errorf("request received was ignored (%v)", err)
http.Error(w, errIncompatibleVersion.Error(), http.StatusPreconditionFailed)
return
}
wcid := h.cid.String()
w.Header().Set("X-Etcd-Cluster-ID", wcid)
gcid := r.Header.Get("X-Etcd-Cluster-ID")
if gcid != wcid {
plog.Errorf("request received was ignored (cluster ID mismatch got %s want %s)", gcid, wcid)
http.Error(w, errClusterIDMismatch.Error(), http.StatusPreconditionFailed)
return
}
// Limit the data size that could be read from the request body, which ensures that read from
// connection will not time out accidentally due to possible block in underlying implementation.
limitedr := pioutil.NewLimitedBufferReader(r.Body, ConnReadLimitByte)
b, err := ioutil.ReadAll(limitedr)
if err != nil {
plog.Errorf("failed to read raft message (%v)", err)
http.Error(w, "error reading raft message", http.StatusBadRequest)
return
}
var m raftpb.Message
if err := m.Unmarshal(b); err != nil {
plog.Errorf("failed to unmarshal raft message (%v)", err)
http.Error(w, "error unmarshaling raft message", http.StatusBadRequest)
return
}
if err := h.r.Process(context.TODO(), m); err != nil {
switch v := err.(type) {
case writerToResponse:
v.WriteTo(w)
default:
plog.Warningf("failed to process raft message (%v)", err)
http.Error(w, "error processing raft message", http.StatusInternalServerError)
}
return
}
// Write StatusNoContet header after the message has been processed by
// raft, which faciliates the client to report MsgSnap status.
w.WriteHeader(http.StatusNoContent)
}
func TestPause(t *testing.T) {
peers := []raft.Peer{{1, nil}, {2, nil}, {3, nil}, {4, nil}, {5, nil}}
nt := newRaftNetwork(1, 2, 3, 4, 5)
nodes := make([]*node, 0)
for i := 1; i <= 5; i++ {
n := startNode(uint64(i), peers, nt.nodeNetwork(uint64(i)))
nodes = append(nodes, n)
}
time.Sleep(50 * time.Millisecond)
for i := 0; i < 300; i++ {
nodes[0].Propose(context.TODO(), []byte("somedata"))
}
nodes[1].pause()
for i := 0; i < 300; i++ {
nodes[0].Propose(context.TODO(), []byte("somedata"))
}
nodes[2].pause()
for i := 0; i < 300; i++ {
nodes[0].Propose(context.TODO(), []byte("somedata"))
}
nodes[2].resume()
for i := 0; i < 300; i++ {
nodes[0].Propose(context.TODO(), []byte("somedata"))
}
nodes[1].resume()
// give some time for nodes to catch up with the raft leader
time.Sleep(300 * time.Millisecond)
for _, n := range nodes {
n.stop()
if n.state.Commit != 1206 {
t.Errorf("commit = %d, want = 1206", n.state.Commit)
}
}
}
// Cancel and Stop should unblock Step()
func TestMultiNodeStepUnblock(t *testing.T) {
// a node without buffer to block step
mn := &multiNode{
propc: make(chan multiMessage),
done: make(chan struct{}),
}
ctx, cancel := context.WithCancel(context.Background())
stopFunc := func() { close(mn.done) }
tests := []struct {
unblock func()
werr error
}{
{stopFunc, ErrStopped},
{cancel, context.Canceled},
}
for i, tt := range tests {
errc := make(chan error, 1)
go func() {
err := mn.Step(ctx, 1, raftpb.Message{Type: raftpb.MsgProp})
errc <- err
}()
tt.unblock()
select {
case err := <-errc:
if err != tt.werr {
t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
}
//clean up side-effect
if ctx.Err() != nil {
ctx = context.TODO()
}
select {
case <-mn.done:
mn.done = make(chan struct{})
default:
}
case <-time.After(time.Millisecond * 100):
t.Errorf("#%d: failed to unblock step", i)
}
}
}
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
w.Header().Set("Allow", "POST")
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
return
}
wcid := h.cid.String()
w.Header().Set("X-Etcd-Cluster-ID", wcid)
gcid := r.Header.Get("X-Etcd-Cluster-ID")
if gcid != wcid {
log.Printf("rafthttp: request ignored due to cluster ID mismatch got %s want %s", gcid, wcid)
http.Error(w, "clusterID mismatch", http.StatusPreconditionFailed)
return
}
// Limit the data size that could be read from the request body, which ensures that read from
// connection will not time out accidentally due to possible block in underlying implementation.
limitedr := pioutil.NewLimitedBufferReader(r.Body, ConnReadLimitByte)
b, err := ioutil.ReadAll(limitedr)
if err != nil {
log.Println("rafthttp: error reading raft message:", err)
http.Error(w, "error reading raft message", http.StatusBadRequest)
return
}
var m raftpb.Message
if err := m.Unmarshal(b); err != nil {
log.Println("rafthttp: error unmarshaling raft message:", err)
http.Error(w, "error unmarshaling raft message", http.StatusBadRequest)
return
}
if err := h.r.Process(context.TODO(), m); err != nil {
switch v := err.(type) {
case writerToResponse:
v.WriteTo(w)
default:
log.Printf("rafthttp: error processing raft message: %v", err)
http.Error(w, "error processing raft message", http.StatusInternalServerError)
}
return
}
w.WriteHeader(http.StatusNoContent)
}
func (s *streamReader) handle(r io.Reader) {
defer func() {
close(s.done)
log.Printf("rafthttp: client streaming to %s at term %d has been stopped", s.to, s.term)
}()
er := newEntryReader(r, s.to)
defer er.stop()
for {
ents, err := er.readEntries()
if err != nil {
if err != io.EOF {
log.Printf("rafthttp: encountered error reading the client log stream: %v", err)
}
return
}
if len(ents) == 0 {
continue
}
// The commit index field in appendEntry message is not recovered.
// The follower updates its commit index through heartbeat.
msg := raftpb.Message{
Type: raftpb.MsgApp,
From: uint64(s.to),
To: uint64(s.id),
Term: s.term,
LogTerm: s.term,
Index: ents[0].Index - 1,
Entries: ents,
}
if err := s.r.Process(context.TODO(), msg); err != nil {
log.Printf("rafthttp: process raft message error: %v", err)
return
}
}
}
"io"
"io/ioutil"
"mime"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"time"
"github.com/coreos/mantle/Godeps/_workspace/src/golang.org/x/net/context"
)
// NoContext is the default context you should supply if not using
// your own context.Context (see https://golang.org/x/net/context).
var NoContext = context.TODO()
// Config describes a typical 3-legged OAuth2 flow, with both the
// client application information and the server's endpoint URLs.
type Config struct {
// ClientID is the application's ID.
ClientID string
// ClientSecret is the application's secret.
ClientSecret string
// Endpoint contains the resource server's token endpoint
// URLs. These are constants specific to each server and are
// often available via site-specific packages, such as
// google.Endpoint or github.Endpoint.
Endpoint Endpoint
func startPeer(tr http.RoundTripper, urls types.URLs, local, to, cid types.ID, r Raft, fs *stats.FollowerStats, errorc chan error, term uint64) *peer {
picker := newURLPicker(urls)
status := newPeerStatus(to)
p := &peer{
id: to,
r: r,
msgAppWriter: startStreamWriter(to, status, fs, r),
writer: startStreamWriter(to, status, fs, r),
pipeline: newPipeline(tr, picker, local, to, cid, status, fs, r, errorc),
sendc: make(chan raftpb.Message),
recvc: make(chan raftpb.Message, recvBufSize),
propc: make(chan raftpb.Message, maxPendingProposals),
newURLsC: make(chan types.URLs),
termc: make(chan uint64),
pausec: make(chan struct{}),
resumec: make(chan struct{}),
stopc: make(chan struct{}),
done: make(chan struct{}),
}
// Use go-routine for process of MsgProp because it is
// blocking when there is no leader.
ctx, cancel := context.WithCancel(context.Background())
go func() {
for {
select {
case mm := <-p.propc:
if err := r.Process(ctx, mm); err != nil {
plog.Warningf("failed to process raft message (%v)", err)
}
case <-p.stopc:
return
}
}
}()
p.msgAppReader = startStreamReader(tr, picker, streamTypeMsgAppV2, local, to, cid, status, p.recvc, p.propc, errorc, term)
reader := startStreamReader(tr, picker, streamTypeMessage, local, to, cid, status, p.recvc, p.propc, errorc, term)
go func() {
var paused bool
for {
select {
case m := <-p.sendc:
if paused {
continue
}
writec, name := p.pick(m)
select {
case writec <- m:
default:
p.r.ReportUnreachable(m.To)
if isMsgSnap(m) {
p.r.ReportSnapshot(m.To, raft.SnapshotFailure)
}
if status.isActive() {
plog.Warningf("dropped %s to %s since %s's sending buffer is full", m.Type, p.id, name)
} else {
plog.Debugf("dropped %s to %s since %s's sending buffer is full", m.Type, p.id, name)
}
}
case mm := <-p.recvc:
if err := r.Process(context.TODO(), mm); err != nil {
plog.Warningf("failed to process raft message (%v)", err)
}
case urls := <-p.newURLsC:
picker.update(urls)
case <-p.pausec:
paused = true
case <-p.resumec:
paused = false
case <-p.stopc:
cancel()
p.msgAppWriter.stop()
p.writer.stop()
p.pipeline.stop()
p.msgAppReader.stop()
reader.stop()
close(p.done)
return
}
}
}()
return p
}
// operateHeader takes action on the decoded headers. It returns the current
// stream if there are remaining headers on the wire (in the following
// Continuation frame).
func (t *http2Server) operateHeaders(hDec *hpackDecoder, s *Stream, frame headerFrame, endStream bool, handle func(*Stream), wg *sync.WaitGroup) (pendingStream *Stream) {
defer func() {
if pendingStream == nil {
hDec.state = decodeState{}
}
}()
endHeaders, err := hDec.decodeServerHTTP2Headers(frame)
if s == nil {
// s has been closed.
return nil
}
if err != nil {
grpclog.Printf("transport: http2Server.operateHeader found %v", err)
if se, ok := err.(StreamError); ok {
t.controlBuf.put(&resetStream{s.id, statusCodeConvTab[se.Code]})
}
return nil
}
if endStream {
// s is just created by the caller. No lock needed.
s.state = streamReadDone
}
if !endHeaders {
return s
}
t.mu.Lock()
if t.state != reachable {
t.mu.Unlock()
return nil
}
if uint32(len(t.activeStreams)) >= t.maxStreams {
t.mu.Unlock()
t.controlBuf.put(&resetStream{s.id, http2.ErrCodeRefusedStream})
return nil
}
s.sendQuotaPool = newQuotaPool(int(t.streamSendQuota))
t.activeStreams[s.id] = s
t.mu.Unlock()
s.windowHandler = func(n int) {
t.updateWindow(s, uint32(n))
}
if hDec.state.timeoutSet {
s.ctx, s.cancel = context.WithTimeout(context.TODO(), hDec.state.timeout)
} else {
s.ctx, s.cancel = context.WithCancel(context.TODO())
}
// Attach Auth info if there is any.
if t.authInfo != nil {
s.ctx = credentials.NewContext(s.ctx, t.authInfo)
}
// Cache the current stream to the context so that the server application
// can find out. Required when the server wants to send some metadata
// back to the client (unary call only).
s.ctx = newContextWithStream(s.ctx, s)
// Attach the received metadata to the context.
if len(hDec.state.mdata) > 0 {
s.ctx = metadata.NewContext(s.ctx, hDec.state.mdata)
}
s.dec = &recvBufferReader{
ctx: s.ctx,
recv: s.buf,
}
s.method = hDec.state.method
wg.Add(1)
go func() {
handle(s)
wg.Done()
}()
return nil
}