func (h *keysHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !allowMethod(w, r.Method, "HEAD", "GET", "PUT", "POST", "DELETE") { return } w.Header().Set("X-Etcd-Cluster-ID", h.clusterInfo.ID().String()) ctx, cancel := context.WithTimeout(context.Background(), h.timeout) defer cancel() rr, err := parseKeyRequest(r, clockwork.NewRealClock()) if err != nil { writeError(w, err) return } resp, err := h.server.Do(ctx, rr) if err != nil { err = trimErrorPrefix(err, etcdserver.StoreKeysPrefix) writeError(w, err) return } switch { case resp.Event != nil: if err := writeKeyEvent(w, resp.Event, h.timer); err != nil { // Should never be reached log.Printf("error writing event: %v", err) } case resp.Watcher != nil: ctx, cancel := context.WithTimeout(context.Background(), defaultWatchTimeout) defer cancel() handleKeyWatch(ctx, w, resp.Watcher, rr.Stream, h.timer) default: writeError(w, errors.New("received response with no Event/Watcher!")) } }
func (d *discovery) checkCluster() ([]*client.Node, int, uint64, error) { configKey := path.Join("/", d.cluster, "_config") ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout) // find cluster size resp, err := d.c.Get(ctx, path.Join(configKey, "size"), nil) cancel() if err != nil { if eerr, ok := err.(*client.Error); ok && eerr.Code == client.ErrorCodeKeyNotFound { return nil, 0, 0, ErrSizeNotFound } if err == client.ErrInvalidJSON { return nil, 0, 0, ErrBadDiscoveryEndpoint } if ce, ok := err.(*client.ClusterError); ok { plog.Error(ce.Detail()) return d.checkClusterRetry() } return nil, 0, 0, err } size, err := strconv.Atoi(resp.Node.Value) if err != nil { return nil, 0, 0, ErrBadSizeKey } ctx, cancel = context.WithTimeout(context.Background(), client.DefaultRequestTimeout) resp, err = d.c.Get(ctx, d.cluster, nil) cancel() if err != nil { if ce, ok := err.(*client.ClusterError); ok { plog.Error(ce.Detail()) return d.checkClusterRetry() } return nil, 0, 0, err } nodes := make([]*client.Node, 0) // append non-config keys to nodes for _, n := range resp.Node.Nodes { if !(path.Base(n.Key) == path.Base(configKey)) { nodes = append(nodes, n) } } snodes := sortableNodes{nodes} sort.Sort(snodes) // find self position for i := range nodes { if path.Base(nodes[i].Key) == path.Base(d.selfKey()) { break } if i >= size-1 { return nodes[:size], size, resp.Index, ErrFullCluster } } return nodes, size, resp.Index, nil }
// publish registers server information into the cluster. The information // is the JSON representation of this server's member struct, updated with the // static clientURLs of the server. // The function keeps attempting to register until it succeeds, // or its server is stopped. func (s *EtcdServer) publish(retryInterval time.Duration) { b, err := json.Marshal(s.attributes) if err != nil { plog.Panicf("json marshal error: %v", err) return } req := pb.Request{ Method: "PUT", Path: MemberAttributesStorePath(s.id), Val: string(b), } for { ctx, cancel := context.WithTimeout(context.Background(), retryInterval) _, err := s.Do(ctx, req) cancel() switch err { case nil: plog.Infof("published %+v to cluster %s", s.attributes, s.cluster.ID()) return case ErrStopped: plog.Infof("aborting publish because server is stopped") return default: plog.Errorf("publish error: %v", err) } } }
func testExceedMaxStreamsLimit(t *testing.T, e env) { // Only allows 1 live stream per server transport. s, cc := setUp(nil, 1, "", e) tc := testpb.NewTestServiceClient(cc) defer tearDown(s, cc) done := make(chan struct{}) ch := make(chan int) go func() { for { select { case <-time.After(5 * time.Millisecond): ch <- 0 case <-time.After(5 * time.Second): close(done) return } } }() // Loop until a stream creation hangs due to the new max stream setting. for { select { case <-ch: ctx, _ := context.WithTimeout(context.Background(), time.Second) if _, err := tc.StreamingInputCall(ctx); err != nil { if grpc.Code(err) == codes.DeadlineExceeded { return } t.Fatalf("%v.StreamingInputCall(_) = %v, want <nil>", tc, err) } case <-done: t.Fatalf("Client has not received the max stream setting in 5 seconds.") } } }
func testTimeoutOnDeadServer(t *testing.T, e env) { s, cc := setUp(nil, math.MaxUint32, "", e) tc := testpb.NewTestServiceClient(cc) if ok := cc.WaitForStateChange(time.Second, grpc.Idle); !ok { t.Fatalf("cc.WaitForStateChange(_, %s) = %t, want true", grpc.Idle, ok) } if ok := cc.WaitForStateChange(time.Second, grpc.Connecting); !ok { t.Fatalf("cc.WaitForStateChange(_, %s) = %t, want true", grpc.Connecting, ok) } if cc.State() != grpc.Ready { t.Fatalf("cc.State() = %s, want %s", cc.State(), grpc.Ready) } if ok := cc.WaitForStateChange(time.Millisecond, grpc.Ready); ok { t.Fatalf("cc.WaitForStateChange(_, %s) = %t, want false", grpc.Ready, ok) } s.Stop() // Set -1 as the timeout to make sure if transportMonitor gets error // notification in time the failure path of the 1st invoke of // ClientConn.wait hits the deadline exceeded error. ctx, _ := context.WithTimeout(context.Background(), -1) if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); grpc.Code(err) != codes.DeadlineExceeded { t.Fatalf("TestService/EmptyCall(%v, _) = _, error %v, want _, error code: %d", ctx, err, codes.DeadlineExceeded) } if ok := cc.WaitForStateChange(time.Second, grpc.Ready); !ok { t.Fatalf("cc.WaitForStateChange(_, %s) = %t, want true", grpc.Ready, ok) } state := cc.State() if state != grpc.Connecting && state != grpc.TransientFailure { t.Fatalf("cc.State() = %s, want %s or %s", state, grpc.Connecting, grpc.TransientFailure) } cc.Close() }
func (s *store) ensureAuthDirectories() error { if s.ensuredOnce { return nil } for _, res := range []string{StorePermsPrefix, StorePermsPrefix + "/users/", StorePermsPrefix + "/roles/"} { ctx, cancel := context.WithTimeout(context.Background(), s.timeout) defer cancel() pe := false rr := etcdserverpb.Request{ Method: "PUT", Path: res, Dir: true, PrevExist: &pe, } _, err := s.server.Do(ctx, rr) if err != nil { if e, ok := err.(*etcderr.Error); ok { if e.ErrorCode == etcderr.EcodeNodeExist { continue } } plog.Errorf("failed to create auth directories in the store (%v)", err) return err } } ctx, cancel := context.WithTimeout(context.Background(), s.timeout) defer cancel() pe := false rr := etcdserverpb.Request{ Method: "PUT", Path: StorePermsPrefix + "/enabled", Val: "false", PrevExist: &pe, } _, err := s.server.Do(ctx, rr) if err != nil { if e, ok := err.(*etcderr.Error); ok { if e.ErrorCode == etcderr.EcodeNodeExist { s.ensuredOnce = true return nil } } return err } s.ensuredOnce = true return nil }
func healthCheck(t time.Duration, cc *grpc.ClientConn, serviceName string) (*healthpb.HealthCheckResponse, error) { ctx, _ := context.WithTimeout(context.Background(), t) hc := healthpb.NewHealthCheckClient(cc) req := &healthpb.HealthCheckRequest{ Service: serviceName, } return hc.Check(ctx, req) }
func (s *store) requestResource(res string, dir bool) (etcdserver.Response, error) { ctx, cancel := context.WithTimeout(context.Background(), s.timeout) defer cancel() p := path.Join(StorePermsPrefix, res) rr := etcdserverpb.Request{ Method: "GET", Path: p, Dir: dir, } return s.server.Do(ctx, rr) }
func (h *keysHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !allowMethod(w, r.Method, "HEAD", "GET", "PUT", "POST", "DELETE") { return } w.Header().Set("X-Etcd-Cluster-ID", h.cluster.ID().String()) ctx, cancel := context.WithTimeout(context.Background(), h.timeout) defer cancel() rr, err := parseKeyRequest(r, clockwork.NewRealClock()) if err != nil { writeKeyError(w, err) return } // The path must be valid at this point (we've parsed the request successfully). if !hasKeyPrefixAccess(h.sec, r, r.URL.Path[len(keysPrefix):], rr.Recursive) { writeKeyNoAuth(w) return } resp, err := h.server.Do(ctx, rr) if err != nil { err = trimErrorPrefix(err, etcdserver.StoreKeysPrefix) writeKeyError(w, err) return } switch { case resp.Event != nil: if err := writeKeyEvent(w, resp.Event, h.timer); err != nil { // Should never be reached plog.Errorf("error writing event (%v)", err) } case resp.Watcher != nil: ctx, cancel := context.WithTimeout(context.Background(), defaultWatchTimeout) defer cancel() handleKeyWatch(ctx, w, resp.Watcher, rr.Stream, h.timer) default: writeKeyError(w, errors.New("received response with no Event/Watcher!")) } }
func ExampleWithTimeout() { // Pass a context with a timeout to tell a blocking function that it // should abandon its work after the timeout elapses. ctx, _ := context.WithTimeout(context.Background(), 100*time.Millisecond) select { case <-time.After(200 * time.Millisecond): fmt.Println("overslept") case <-ctx.Done(): fmt.Println(ctx.Err()) // prints "context deadline exceeded" } // Output: // context deadline exceeded }
// sync proposes a SYNC request and is non-blocking. // This makes no guarantee that the request will be proposed or performed. // The request will be cancelled after the given timeout. func (s *EtcdServer) sync(timeout time.Duration) { ctx, cancel := context.WithTimeout(context.Background(), timeout) req := pb.Request{ Method: "SYNC", ID: s.reqIDGen.Next(), Time: time.Now().UnixNano(), } data := pbutil.MustMarshal(&req) // There is no promise that node has leader when do SYNC request, // so it uses goroutine to propose. go func() { s.r.Propose(ctx, data) cancel() }() }
func (d *discovery) createSelf(contents string) error { ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout) resp, err := d.c.Create(ctx, d.selfKey(), contents) cancel() if err != nil { if eerr, ok := err.(client.Error); ok && eerr.Code == client.ErrorCodeNodeExist { return ErrDuplicateID } return err } // ensure self appears on the server we connected to w := d.c.Watcher(d.selfKey(), &client.WatcherOptions{AfterIndex: resp.Node.CreatedIndex - 1}) _, err = w.Next(context.Background()) return err }
func (d *discovery) createSelf(contents string) error { ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout) resp, err := d.c.Create(ctx, d.selfKey(), contents, -1) cancel() if err != nil { if err == client.ErrKeyExists { return ErrDuplicateID } return err } // ensure self appears on the server we connected to w := d.c.Watch(d.selfKey(), resp.Node.CreatedIndex) _, err = w.Next(context.Background()) return err }
func (s *store) deleteResource(res string) (etcdserver.Response, error) { err := s.ensureAuthDirectories() if err != nil { return etcdserver.Response{}, err } ctx, cancel := context.WithTimeout(context.Background(), s.timeout) defer cancel() pex := true p := path.Join(StorePermsPrefix, res) rr := etcdserverpb.Request{ Method: "DELETE", Path: p, PrevExist: &pex, } return s.server.Do(ctx, rr) }
// TODO(zhaoq): Have a better test coverage of timeout and cancellation mechanism. func testRPCTimeout(t *testing.T, e env) { s, cc := setUp(nil, math.MaxUint32, "", e) tc := testpb.NewTestServiceClient(cc) defer tearDown(s, cc) argSize := 2718 respSize := 314 req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(), ResponseSize: proto.Int32(int32(respSize)), Payload: newPayload(testpb.PayloadType_COMPRESSABLE, int32(argSize)), } for i := -1; i <= 10; i++ { ctx, _ := context.WithTimeout(context.Background(), time.Duration(i)*time.Millisecond) reply, err := tc.UnaryCall(ctx, req) if grpc.Code(err) != codes.DeadlineExceeded { t.Fatalf(`TestService/UnaryCallv(_, _) = %v, %v; want <nil>, error code: %d`, reply, err, codes.DeadlineExceeded) } } }
func (s *store) setResource(res string, value interface{}, prevexist bool) (etcdserver.Response, error) { err := s.ensureAuthDirectories() if err != nil { return etcdserver.Response{}, err } ctx, cancel := context.WithTimeout(context.Background(), s.timeout) defer cancel() data, err := json.Marshal(value) if err != nil { return etcdserver.Response{}, err } p := path.Join(StorePermsPrefix, res) rr := etcdserverpb.Request{ Method: "PUT", Path: p, Val: string(data), PrevExist: &prevexist, } return s.server.Do(ctx, rr) }
func TestLargeMessageSuspension(t *testing.T) { server, ct := setUp(t, 0, math.MaxUint32, suspended) callHdr := &CallHdr{ Host: "localhost", Method: "foo.Large", } // Set a long enough timeout for writing a large message out. ctx, _ := context.WithTimeout(context.Background(), time.Second) s, err := ct.NewStream(ctx, callHdr) if err != nil { t.Fatalf("failed to open stream: %v", err) } // Write should not be done successfully due to flow control. err = ct.Write(s, expectedRequestLarge, &Options{Last: true, Delay: false}) expectedErr := StreamErrorf(codes.DeadlineExceeded, "%v", context.DeadlineExceeded) if err == nil || err != expectedErr { t.Fatalf("Write got %v, want %v", err, expectedErr) } ct.Close() server.stop() }
func doTimeoutOnSleepingServer(tc testpb.TestServiceClient) { ctx, _ := context.WithTimeout(context.Background(), 1*time.Millisecond) stream, err := tc.FullDuplexCall(ctx) if err != nil { if grpc.Code(err) == codes.DeadlineExceeded { grpclog.Println("TimeoutOnSleepingServer done") return } grpclog.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err) } pl := newPayload(testpb.PayloadType_COMPRESSABLE, 27182) req := &testpb.StreamingOutputCallRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(), Payload: pl, } if err := stream.Send(req); err != nil { grpclog.Fatalf("%v.Send(%v) = %v", stream, req, err) } if _, err := stream.Recv(); grpc.Code(err) != codes.DeadlineExceeded { grpclog.Fatalf("%v.Recv() = _, %v, want error code %d", stream, err, codes.DeadlineExceeded) } grpclog.Println("TimeoutOnSleepingServer done") }
func (s *EtcdServer) updateClusterVersion(ver string) { if s.cluster.Version() == nil { plog.Infof("setting up the initial cluster version to %v", ver) } else { plog.Infof("updating the cluster version from %v to %v", s.cluster.Version(), ver) } req := pb.Request{ Method: "PUT", Path: path.Join(StoreClusterPrefix, "version"), Val: ver, } ctx, cancel := context.WithTimeout(context.Background(), versionUpdateTimeout) _, err := s.Do(ctx, req) cancel() switch err { case nil: return case ErrStopped: plog.Infof("aborting update cluster version because server is stopped") return default: plog.Errorf("error updating cluster version (%v)", err) } }
func (c *simpleHTTPClient) Do(ctx context.Context, act httpAction) (*http.Response, []byte, error) { req := act.HTTPRequest(c.endpoint) if err := printcURL(req); err != nil { return nil, nil, err } hctx, hcancel := context.WithCancel(ctx) if c.headerTimeout > 0 { hctx, hcancel = context.WithTimeout(ctx, c.headerTimeout) } defer hcancel() reqcancel := requestCanceler(c.transport, req) rtchan := make(chan roundTripResponse, 1) go func() { resp, err := c.transport.RoundTrip(req) rtchan <- roundTripResponse{resp: resp, err: err} close(rtchan) }() var resp *http.Response var err error select { case rtresp := <-rtchan: resp, err = rtresp.resp, rtresp.err case <-hctx.Done(): // cancel and wait for request to actually exit before continuing reqcancel() rtresp := <-rtchan resp = rtresp.resp switch { case ctx.Err() != nil: err = ctx.Err() case hctx.Err() != nil: err = fmt.Errorf("client: endpoint %s exceeded header timeout", c.endpoint.String()) default: panic("failed to get error from context") } } // always check for resp nil-ness to deal with possible // race conditions between channels above defer func() { if resp != nil { resp.Body.Close() } }() if err != nil { return nil, nil, err } var body []byte done := make(chan struct{}) go func() { body, err = ioutil.ReadAll(resp.Body) done <- struct{}{} }() select { case <-ctx.Done(): resp.Body.Close() <-done return nil, nil, ctx.Err() case <-done: } return resp, body, err }
func (h *membersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !allowMethod(w, r.Method, "GET", "POST", "DELETE", "PUT") { return } w.Header().Set("X-Etcd-Cluster-ID", h.clusterInfo.ID().String()) ctx, cancel := context.WithTimeout(context.Background(), defaultServerTimeout) defer cancel() switch r.Method { case "GET": switch trimPrefix(r.URL.Path, membersPrefix) { case "": mc := newMemberCollection(h.clusterInfo.Members()) w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(mc); err != nil { log.Printf("etcdhttp: %v", err) } case "leader": id := h.server.Leader() if id == 0 { writeError(w, httptypes.NewHTTPError(http.StatusServiceUnavailable, "During election")) return } m := newMember(h.clusterInfo.Member(id)) w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(m); err != nil { log.Printf("etcdhttp: %v", err) } default: writeError(w, httptypes.NewHTTPError(http.StatusNotFound, "Not found")) } case "POST": req := httptypes.MemberCreateRequest{} if ok := unmarshalRequest(r, &req, w); !ok { return } now := h.clock.Now() m := etcdserver.NewMember("", req.PeerURLs, "", &now) err := h.server.AddMember(ctx, *m) switch { case err == etcdserver.ErrIDExists || err == etcdserver.ErrPeerURLexists: writeError(w, httptypes.NewHTTPError(http.StatusConflict, err.Error())) return case err != nil: log.Printf("etcdhttp: error adding node %s: %v", m.ID, err) writeError(w, err) return } res := newMember(m) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusCreated) if err := json.NewEncoder(w).Encode(res); err != nil { log.Printf("etcdhttp: %v", err) } case "DELETE": id, ok := getID(r.URL.Path, w) if !ok { return } err := h.server.RemoveMember(ctx, uint64(id)) switch { case err == etcdserver.ErrIDRemoved: writeError(w, httptypes.NewHTTPError(http.StatusGone, fmt.Sprintf("Member permanently removed: %s", id))) case err == etcdserver.ErrIDNotFound: writeError(w, httptypes.NewHTTPError(http.StatusNotFound, fmt.Sprintf("No such member: %s", id))) case err != nil: log.Printf("etcdhttp: error removing node %s: %v", id, err) writeError(w, err) default: w.WriteHeader(http.StatusNoContent) } case "PUT": id, ok := getID(r.URL.Path, w) if !ok { return } req := httptypes.MemberUpdateRequest{} if ok := unmarshalRequest(r, &req, w); !ok { return } m := etcdserver.Member{ ID: id, RaftAttributes: etcdserver.RaftAttributes{PeerURLs: req.PeerURLs.StringSlice()}, } err := h.server.UpdateMember(ctx, m) switch { case err == etcdserver.ErrPeerURLexists: writeError(w, httptypes.NewHTTPError(http.StatusConflict, err.Error())) case err == etcdserver.ErrIDNotFound: writeError(w, httptypes.NewHTTPError(http.StatusNotFound, fmt.Sprintf("No such member: %s", id))) case err != nil: log.Printf("etcdhttp: error updating node %s: %v", m.ID, err) writeError(w, err) default: w.WriteHeader(http.StatusNoContent) } } }
// operateHeader takes action on the decoded headers. It returns the current // stream if there are remaining headers on the wire (in the following // Continuation frame). func (t *http2Server) operateHeaders(hDec *hpackDecoder, s *Stream, frame headerFrame, endStream bool, handle func(*Stream), wg *sync.WaitGroup) (pendingStream *Stream) { defer func() { if pendingStream == nil { hDec.state = decodeState{} } }() endHeaders, err := hDec.decodeServerHTTP2Headers(frame) if s == nil { // s has been closed. return nil } if err != nil { grpclog.Printf("transport: http2Server.operateHeader found %v", err) if se, ok := err.(StreamError); ok { t.controlBuf.put(&resetStream{s.id, statusCodeConvTab[se.Code]}) } return nil } if endStream { // s is just created by the caller. No lock needed. s.state = streamReadDone } if !endHeaders { return s } t.mu.Lock() if t.state != reachable { t.mu.Unlock() return nil } if uint32(len(t.activeStreams)) >= t.maxStreams { t.mu.Unlock() t.controlBuf.put(&resetStream{s.id, http2.ErrCodeRefusedStream}) return nil } s.sendQuotaPool = newQuotaPool(int(t.streamSendQuota)) t.activeStreams[s.id] = s t.mu.Unlock() s.windowHandler = func(n int) { t.updateWindow(s, uint32(n)) } if hDec.state.timeoutSet { s.ctx, s.cancel = context.WithTimeout(context.TODO(), hDec.state.timeout) } else { s.ctx, s.cancel = context.WithCancel(context.TODO()) } // Attach Auth info if there is any. if t.authInfo != nil { s.ctx = credentials.NewContext(s.ctx, t.authInfo) } // Cache the current stream to the context so that the server application // can find out. Required when the server wants to send some metadata // back to the client (unary call only). s.ctx = newContextWithStream(s.ctx, s) // Attach the received metadata to the context. if len(hDec.state.mdata) > 0 { s.ctx = metadata.NewContext(s.ctx, hDec.state.mdata) } s.dec = &recvBufferReader{ ctx: s.ctx, recv: s.buf, } s.method = hDec.state.method wg.Add(1) go func() { handle(s) wg.Done() }() return nil }