func (h serverHandler) serveKeys(w http.ResponseWriter, r *http.Request) {
if !allowMethod(w, r.Method, "GET", "PUT", "POST", "DELETE") {
return
}
ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
defer cancel()
rr, err := parseRequest(r, etcdserver.GenID())
if err != nil {
writeError(w, err)
return
}
resp, err := h.server.Do(ctx, rr)
if err != nil {
writeError(w, err)
return
}
switch {
case resp.Event != nil:
if err := writeEvent(w, resp.Event, h.timer); err != nil {
// Should never be reached
log.Printf("error writing event: %v", err)
}
case resp.Watcher != nil:
ctx, cancel := context.WithTimeout(context.Background(), defaultWatchTimeout)
defer cancel()
handleWatch(ctx, w, resp.Watcher, rr.Stream, h.timer)
default:
writeError(w, errors.New("received response with no Event/Watcher!"))
}
}
func TestNode(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
wants := []Ready{
{
SoftState: &SoftState{Lead: 1, RaftState: StateLeader},
HardState: raftpb.HardState{Term: 1, Commit: 1},
Entries: []raftpb.Entry{{}, {Term: 1, Index: 1}},
CommittedEntries: []raftpb.Entry{{Term: 1, Index: 1}},
},
{
HardState: raftpb.HardState{Term: 1, Commit: 2},
Entries: []raftpb.Entry{{Term: 1, Index: 2, Data: []byte("foo")}},
CommittedEntries: []raftpb.Entry{{Term: 1, Index: 2, Data: []byte("foo")}},
},
}
n := Start(1, []int64{1}, 0, 0)
n.Campaign(ctx)
if g := <-n.Ready(); !reflect.DeepEqual(g, wants[0]) {
t.Errorf("#%d: g = %+v,\n w %+v", 1, g, wants[0])
}
n.Propose(ctx, []byte("foo"))
if g := <-n.Ready(); !reflect.DeepEqual(g, wants[1]) {
t.Errorf("#%d: g = %+v,\n w %+v", 2, g, wants[1])
}
select {
case rd := <-n.Ready():
t.Errorf("unexpected Ready: %+v", rd)
default:
}
}
// publish registers server information into the cluster. The information
// is the JSON representation of this server's member struct, updated with the
// static clientURLs of the server.
// The function keeps attempting to register until it succeeds,
// or its server is stopped.
// TODO: take care of info fetched from cluster store after having reconfig.
func (s *EtcdServer) publish(retryInterval time.Duration) {
m := *s.ClusterStore.Get().FindName(s.name)
m.ClientURLs = s.clientURLs.StringSlice()
b, err := json.Marshal(m)
if err != nil {
log.Printf("etcdserver: json marshal error: %v", err)
return
}
req := pb.Request{
ID: GenID(),
Method: "PUT",
Path: m.storeKey(),
Val: string(b),
}
for {
ctx, cancel := context.WithTimeout(context.Background(), retryInterval)
_, err := s.Do(ctx, req)
cancel()
switch err {
case nil:
log.Printf("etcdserver: published %+v to the cluster", m)
return
case ErrStopped:
log.Printf("etcdserver: aborting publish because server is stopped")
return
default:
log.Printf("etcdserver: publish error: %v", err)
}
}
}
func TestHTTPClientDoCancelContextWaitForRoundTrip(t *testing.T) {
tr := newFakeTransport()
c := &httpClient{transport: tr}
donechan := make(chan struct{})
ctx, cancel := context.WithCancel(context.Background())
go func() {
c.do(ctx, &fakeAction{})
close(donechan)
}()
// This should call CancelRequest and begin the cancellation process
cancel()
select {
case <-donechan:
t.Fatalf("httpClient.do should not have exited yet")
default:
}
tr.finishCancel <- struct{}{}
select {
case <-donechan:
//expected behavior
return
case <-time.After(time.Second):
t.Fatalf("httpClient.do did not exit within 1s")
}
}
func TestHTTPClientDoError(t *testing.T) {
tr := newFakeTransport()
c := &httpClient{transport: tr}
tr.errchan <- errors.New("fixture")
_, _, err := c.do(context.Background(), &fakeAction{})
if err == nil {
t.Fatalf("expected non-nil error, got nil")
}
}
func TestHTTPClientDoCancelContext(t *testing.T) {
tr := newFakeTransport()
c := &httpClient{transport: tr}
tr.startCancel <- struct{}{}
tr.finishCancel <- struct{}{}
_, _, err := c.do(context.Background(), &fakeAction{})
if err == nil {
t.Fatalf("expected non-nil error, got nil")
}
}
func ExampleWithTimeout() {
// Pass a context with a timeout to tell a blocking function that it
// should abandon its work after the timeout elapses.
ctx, _ := context.WithTimeout(context.Background(), 100*time.Millisecond)
select {
case <-time.After(200 * time.Millisecond):
fmt.Println("overslept")
case <-ctx.Done():
fmt.Println(ctx.Err()) // prints "context deadline exceeded"
}
// Output:
// context deadline exceeded
}
func TestSet(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
st := store.New()
n := raft.Start(1, []int64{1}, 0, 0)
n.Campaign(ctx)
srv := &etcdserver.Server{
Node: n,
Store: st,
Send: etcdserver.SendFunc(nopSend),
Save: func(st raftpb.State, ents []raftpb.Entry) {},
}
etcdserver.Start(srv)
defer srv.Stop()
h := Handler{
Timeout: time.Hour,
Server: srv,
}
s := httptest.NewServer(h)
defer s.Close()
resp, err := http.PostForm(s.URL+"/v2/keys/foo", url.Values{"value": {"bar"}})
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != 201 {
t.Errorf("StatusCode = %d, expected %d", 201, resp.StatusCode)
}
g := new(store.Event)
if err := json.NewDecoder(resp.Body).Decode(&g); err != nil {
t.Fatal(err)
}
w := &store.NodeExtern{
Key: "/foo/1",
Value: stringp("bar"),
ModifiedIndex: 1,
CreatedIndex: 1,
}
if !reflect.DeepEqual(g.Node, w) {
t.Errorf("g = %+v, want %+v", g.Node, w)
}
}
func (hw *httpWatcher) Next() (*Response, error) {
httpresp, body, err := hw.httpClient.do(context.Background(), &hw.nextWait)
if err != nil {
return nil, err
}
resp, err := unmarshalHTTPResponse(httpresp.StatusCode, body)
if err != nil {
return nil, err
}
hw.nextWait.WaitIndex = resp.Node.ModifiedIndex + 1
return resp, nil
}
func (c *httpClient) Get(key string) (*Response, error) {
get := &getAction{
Key: key,
Recursive: false,
}
ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
httpresp, body, err := c.do(ctx, get)
cancel()
if err != nil {
return nil, err
}
return unmarshalHTTPResponse(httpresp.StatusCode, body)
}
// TestCompacts ensures Node.Compact creates a correct raft snapshot and compacts
// the raft log (call raft.compact)
func TestCompact(t *testing.T) {
ctx := context.Background()
n := newNode()
r := newRaft(1, []int64{1}, 0, 0)
go n.run(r)
n.Campaign(ctx)
n.Propose(ctx, []byte("foo"))
w := raftpb.Snapshot{
Term: 1,
Index: 2, // one nop + one proposal
Data: []byte("a snapshot"),
Nodes: []int64{1},
}
pkg.ForceGosched()
select {
case <-n.Ready():
default:
t.Fatalf("unexpected proposal failure: unable to commit entry")
}
n.Compact(w.Data)
pkg.ForceGosched()
select {
case rd := <-n.Ready():
if !reflect.DeepEqual(rd.Snapshot, w) {
t.Errorf("snap = %+v, want %+v", rd.Snapshot, w)
}
default:
t.Fatalf("unexpected compact failure: unable to create a snapshot")
}
pkg.ForceGosched()
// TODO: this test the run updates the snapi correctly... should be tested
// separately with other kinds of updates
select {
case <-n.Ready():
t.Fatalf("unexpected more ready")
default:
}
n.Stop()
if r.raftLog.offset != w.Index {
t.Errorf("log.offset = %d, want %d", r.raftLog.offset, w.Index)
}
}
func (c *httpClient) Create(key, val string, ttl time.Duration) (*Response, error) {
uintTTL := uint64(ttl.Seconds())
create := &createAction{
Key: key,
Value: val,
TTL: &uintTTL,
}
ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
httpresp, body, err := c.do(ctx, create)
cancel()
if err != nil {
return nil, err
}
return unmarshalHTTPResponse(httpresp.StatusCode, body)
}
// Cancel and Stop should unblock Step()
func TestNodeStepUnblock(t *testing.T) {
// a node without buffer to block step
n := &Node{
propc: make(chan raftpb.Message),
done: make(chan struct{}),
}
ctx, cancel := context.WithCancel(context.Background())
stopFunc := func() { close(n.done) }
tests := []struct {
unblock func()
werr error
}{
{stopFunc, ErrStopped},
{cancel, context.Canceled},
}
for i, tt := range tests {
errc := make(chan error, 1)
go func() {
err := n.Step(ctx, raftpb.Message{Type: msgProp})
errc <- err
}()
tt.unblock()
select {
case err := <-errc:
if err != tt.werr {
t.Errorf("#%d: err = %v, want %v", err, tt.werr)
}
//clean up side-effect
if ctx.Err() != nil {
ctx = context.TODO()
}
select {
case <-n.done:
n.done = make(chan struct{})
default:
}
case <-time.After(time.Millisecond * 100):
t.Errorf("#%d: failed to unblock step", i)
}
}
}
// sync proposes a SYNC request and is non-blocking.
// This makes no guarantee that the request will be proposed or performed.
// The request will be cancelled after the given timeout.
func (s *EtcdServer) sync(timeout time.Duration) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
req := pb.Request{
Method: "SYNC",
Id: GenID(),
Time: time.Now().UnixNano(),
}
data, err := req.Marshal()
if err != nil {
log.Printf("marshal request %#v error: %v", req, err)
return
}
// There is no promise that node has leader when do SYNC request,
// so it uses goroutine to propose.
go func() {
s.Node.Propose(ctx, data)
cancel()
}()
}
func TestWaitForEventCancelledContext(t *testing.T) {
cctx, cancel := context.WithCancel(context.Background())
dw := &dummyWatcher{}
w := httptest.NewRecorder()
var wg sync.WaitGroup
var ev *store.Event
var err error
wg.Add(1)
go func() {
ev, err = waitForEvent(cctx, w, dw)
wg.Done()
}()
cancel()
wg.Wait()
if ev != nil {
t.Fatalf("non-nil Event returned with cancelled context: %v", ev)
}
if err == nil {
t.Fatalf("nil err returned with cancelled context!")
}
}
func (h serverHandler) serveKeys(w http.ResponseWriter, r *http.Request) {
if !allowMethod(w, r.Method, "GET", "PUT", "POST", "DELETE") {
return
}
ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
defer cancel()
rr, err := parseRequest(r, etcdserver.GenID())
if err != nil {
writeError(w, err)
return
}
resp, err := h.server.Do(ctx, rr)
if err != nil {
writeError(w, err)
return
}
var ev *store.Event
switch {
case resp.Event != nil:
ev = resp.Event
case resp.Watcher != nil:
if ev, err = waitForEvent(ctx, w, resp.Watcher); err != nil {
http.Error(w, err.Error(), http.StatusGatewayTimeout)
return
}
default:
writeError(w, errors.New("received response with no Event/Watcher!"))
return
}
if err = writeEvent(w, ev); err != nil {
// Should never be reached
log.Println("error writing event: %v", err)
}
}
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// TODO: set read/write timeout?
timeout := h.Timeout
if timeout == 0 {
timeout = DefaultTimeout
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
switch {
case strings.HasPrefix(r.URL.Path, "/raft"):
h.serveRaft(ctx, w, r)
case strings.HasPrefix(r.URL.Path, keysPrefix):
h.serveKeys(ctx, w, r)
case strings.HasPrefix(r.URL.Path, machinesPrefix):
h.serveMachines(w, r)
default:
http.NotFound(w, r)
}
}
func TestWaitForEventChan(t *testing.T) {
ctx := context.Background()
ec := make(chan *store.Event)
dw := &dummyWatcher{
echan: ec,
}
w := httptest.NewRecorder()
var wg sync.WaitGroup
var ev *store.Event
var err error
wg.Add(1)
go func() {
ev, err = waitForEvent(ctx, w, dw)
wg.Done()
}()
ec <- &store.Event{
Action: store.Get,
Node: &store.NodeExtern{
Key: "/foo/bar",
ModifiedIndex: 12345,
},
}
wg.Wait()
want := &store.Event{
Action: store.Get,
Node: &store.NodeExtern{
Key: "/foo/bar",
ModifiedIndex: 12345,
},
}
if !reflect.DeepEqual(ev, want) {
t.Fatalf("bad event: got %#v, want %#v", ev, want)
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestHTTPClientDoSuccess(t *testing.T) {
tr := newFakeTransport()
c := &httpClient{transport: tr}
tr.respchan <- &http.Response{
StatusCode: http.StatusTeapot,
Body: ioutil.NopCloser(strings.NewReader("foo")),
}
resp, body, err := c.do(context.Background(), &fakeAction{})
if err != nil {
t.Fatalf("incorrect error value: want=nil got=%v", err)
}
wantCode := http.StatusTeapot
if wantCode != resp.StatusCode {
t.Fatalf("invalid response code: want=%d got=%d", wantCode, resp.StatusCode)
}
wantBody := []byte("foo")
if !reflect.DeepEqual(wantBody, body) {
t.Fatalf("invalid response body: want=%q got=%q", wantBody, body)
}
}
func TestWaitForEventCloseNotify(t *testing.T) {
ctx := context.Background()
dw := &dummyWatcher{}
cnchan := make(chan bool)
w := &dummyResponseWriter{
cnchan: cnchan,
}
var wg sync.WaitGroup
var ev *store.Event
var err error
wg.Add(1)
go func() {
ev, err = waitForEvent(ctx, w, dw)
wg.Done()
}()
close(cnchan)
wg.Wait()
if ev != nil {
t.Fatalf("non-nil Event returned with CloseNotifier: %v", ev)
}
if err == nil {
t.Fatalf("nil err returned with CloseNotifier!")
}
}
func testServer(t *testing.T, ns int64) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ss := make([]*Server, ns)
send := func(msgs []raftpb.Message) {
for _, m := range msgs {
t.Logf("m = %+v\n", m)
ss[m.To-1].Node.Step(ctx, m)
}
}
peers := make([]int64, ns)
for i := int64(0); i < ns; i++ {
peers[i] = i + 1
}
for i := int64(0); i < ns; i++ {
id := i + 1
n := raft.Start(id, peers, 10, 1)
tk := time.NewTicker(10 * time.Millisecond)
defer tk.Stop()
srv := &Server{
Node: n,
Store: store.New(),
Send: send,
Save: func(_ raftpb.State, _ []raftpb.Entry) {},
Ticker: tk.C,
}
Start(srv)
// TODO(xiangli): randomize election timeout
// then remove this sleep.
time.Sleep(1 * time.Millisecond)
ss[i] = srv
}
for i := 1; i <= 10; i++ {
r := pb.Request{
Method: "PUT",
Id: int64(i),
Path: "/foo",
Val: "bar",
}
j := rand.Intn(len(ss))
t.Logf("ss = %d", j)
resp, err := ss[j].Do(ctx, r)
if err != nil {
t.Fatal(err)
}
g, w := resp.Event.Node, &store.NodeExtern{
Key: "/foo",
ModifiedIndex: uint64(i),
CreatedIndex: uint64(i),
Value: stringp("bar"),
}
if !reflect.DeepEqual(g, w) {
t.Error("value:", *g.Value)
t.Errorf("g = %+v, w %+v", g, w)
}
}
time.Sleep(10 * time.Millisecond)
var last interface{}
for i, sv := range ss {
sv.Stop()
g, _ := sv.Store.Get("/", true, true)
if last != nil && !reflect.DeepEqual(last, g) {
t.Errorf("server %d: Root = %#v, want %#v", i, g, last)
}
last = g
}
}