// TimeToLiveHTTP retrieves lease information of the given lease ID.
func TimeToLiveHTTP(ctx context.Context, id lease.LeaseID, keys bool, url string, rt http.RoundTripper) (*leasepb.LeaseInternalResponse, error) {
// will post lreq protobuf to leader
lreq, err := (&leasepb.LeaseInternalRequest{&pb.LeaseTimeToLiveRequest{ID: int64(id), Keys: keys}}).Marshal()
if err != nil {
return nil, err
}
req, err := http.NewRequest("POST", url, bytes.NewReader(lreq))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/protobuf")
cancel := httputil.RequestCanceler(req)
cc := &http.Client{Transport: rt}
var b []byte
errc := make(chan error)
go func() {
// TODO detect if leader failed and retry?
resp, err := cc.Do(req)
if err != nil {
errc <- err
return
}
b, err = ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
errc <- err
return
}
if resp.StatusCode == http.StatusNotFound {
errc <- lease.ErrLeaseNotFound
return
}
if resp.StatusCode != http.StatusOK {
errc <- fmt.Errorf("lease: unknown error(%s)", string(b))
return
}
errc <- nil
}()
select {
case derr := <-errc:
if derr != nil {
return nil, derr
}
case <-ctx.Done():
cancel()
return nil, ctx.Err()
}
lresp := &leasepb.LeaseInternalResponse{}
if err := lresp.Unmarshal(b); err != nil {
return nil, fmt.Errorf(`lease: %v. data = "%s"`, err, string(b))
}
if lresp.LeaseTimeToLiveResponse.ID != int64(id) {
return nil, fmt.Errorf("lease: renew id mismatch")
}
return lresp, nil
}
// post POSTs a data payload to a url. Returns nil if the POST succeeds,
// error on any failure.
func (p *pipeline) post(data []byte) (err error) {
u := p.picker.pick()
req := createPostRequest(u, RaftPrefix, bytes.NewBuffer(data), "application/protobuf", p.from, p.cid)
var stopped bool
defer func() {
if stopped {
// rewrite to errStopped so the caller goroutine can stop itself
err = errStopped
}
}()
done := make(chan struct{}, 1)
cancel := httputil.RequestCanceler(p.tr, req)
go func() {
select {
case <-done:
case <-p.stopc:
waitSchedule()
stopped = true
cancel()
}
}()
resp, err := p.tr.RoundTrip(req)
done <- struct{}{}
if err != nil {
p.picker.unreachable(u)
return err
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
p.picker.unreachable(u)
return err
}
resp.Body.Close()
err = checkPostResponse(resp, b, req, p.to)
// errMemberRemoved is a critical error since a removed member should
// always be stopped. So we use reportCriticalError to report it to errorc.
if err == errMemberRemoved {
reportCriticalError(err, p.errorc)
return nil
}
return err
}
// post POSTs a data payload to a url. Returns nil if the POST succeeds,
// error on any failure.
func (p *pipeline) post(data []byte) (err error) {
u := p.picker.pick()
req := createPostRequest(u, RaftPrefix, bytes.NewBuffer(data), "application/protobuf", p.tr.URLs, p.tr.ID, p.tr.ClusterID)
done := make(chan struct{}, 1)
cancel := httputil.RequestCanceler(req)
go func() {
select {
case <-done:
case <-p.stopc:
waitSchedule()
cancel()
}
}()
resp, err := p.tr.pipelineRt.RoundTrip(req)
done <- struct{}{}
if err != nil {
p.picker.unreachable(u)
return err
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
p.picker.unreachable(u)
return err
}
resp.Body.Close()
err = checkPostResponse(resp, b, req, p.peerID)
if err != nil {
p.picker.unreachable(u)
// errMemberRemoved is a critical error since a removed member should
// always be stopped. So we use reportCriticalError to report it to errorc.
if err == errMemberRemoved {
reportCriticalError(err, p.errorc)
}
return err
}
return nil
}
// post posts the given request.
// It returns nil when request is sent out and processed successfully.
func (s *snapshotSender) post(req *http.Request) (err error) {
cancel := httputil.RequestCanceler(s.tr, req)
type responseAndError struct {
resp *http.Response
body []byte
err error
}
result := make(chan responseAndError, 1)
go func() {
// TODO: cancel the request if it has waited for a long time(~5s) after
// it has write out the full request body, which helps to avoid receiver
// dies when sender is waiting for response
// TODO: the snapshot could be large and eat up all resources when writing
// it out. Send it block by block and rest some time between to give the
// time for main loop to run.
resp, err := s.tr.RoundTrip(req)
if err != nil {
result <- responseAndError{resp, nil, err}
return
}
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
result <- responseAndError{resp, body, err}
}()
select {
case <-s.stopc:
cancel()
return errStopped
case r := <-result:
if r.err != nil {
return r.err
}
return checkPostResponse(r.resp, r.body, req, s.to)
}
}
// post posts the given request.
// It returns nil when request is sent out and processed successfully.
func (s *snapshotSender) post(req *http.Request) (err error) {
cancel := httputil.RequestCanceler(req)
type responseAndError struct {
resp *http.Response
body []byte
err error
}
result := make(chan responseAndError, 1)
go func() {
resp, err := s.tr.pipelineRt.RoundTrip(req)
if err != nil {
result <- responseAndError{resp, nil, err}
return
}
// close the response body when timeouts.
// prevents from reading the body forever when the other side dies right after
// successfully receives the request body.
time.AfterFunc(snapResponseReadTimeout, func() { httputil.GracefulClose(resp) })
body, err := ioutil.ReadAll(resp.Body)
result <- responseAndError{resp, body, err}
}()
select {
case <-s.stopc:
cancel()
return errStopped
case r := <-result:
if r.err != nil {
return r.err
}
return checkPostResponse(r.resp, r.body, req, s.to)
}
}
func (cr *streamReader) dial(t streamType) (io.ReadCloser, error) {
u := cr.picker.pick()
uu := u
uu.Path = path.Join(t.endpoint(), cr.local.String())
req, err := http.NewRequest("GET", uu.String(), nil)
if err != nil {
cr.picker.unreachable(u)
return nil, fmt.Errorf("failed to make http request to %s (%v)", u, err)
}
req.Header.Set("X-Server-From", cr.local.String())
req.Header.Set("X-Server-Version", version.Version)
req.Header.Set("X-Min-Cluster-Version", version.MinClusterVersion)
req.Header.Set("X-Etcd-Cluster-ID", cr.cid.String())
req.Header.Set("X-Raft-To", cr.remote.String())
cr.mu.Lock()
select {
case <-cr.stopc:
cr.mu.Unlock()
return nil, fmt.Errorf("stream reader is stopped")
default:
}
cr.cancel = httputil.RequestCanceler(cr.tr, req)
cr.mu.Unlock()
resp, err := cr.tr.RoundTrip(req)
if err != nil {
cr.picker.unreachable(u)
return nil, err
}
rv := serverVersion(resp.Header)
lv := semver.Must(semver.NewVersion(version.Version))
if compareMajorMinorVersion(rv, lv) == -1 && !checkStreamSupport(rv, t) {
resp.Body.Close()
return nil, errUnsupportedStreamType
}
switch resp.StatusCode {
case http.StatusGone:
resp.Body.Close()
err := fmt.Errorf("the member has been permanently removed from the cluster")
select {
case cr.errorc <- err:
default:
}
return nil, err
case http.StatusOK:
return resp.Body, nil
case http.StatusNotFound:
resp.Body.Close()
return nil, fmt.Errorf("remote member %s could not recognize local member", cr.remote)
case http.StatusPreconditionFailed:
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
cr.picker.unreachable(u)
return nil, err
}
resp.Body.Close()
switch strings.TrimSuffix(string(b), "\n") {
case errIncompatibleVersion.Error():
plog.Errorf("request sent was ignored by peer %s (server version incompatible)", cr.remote)
return nil, errIncompatibleVersion
case errClusterIDMismatch.Error():
plog.Errorf("request sent was ignored (cluster ID mismatch: remote[%s]=%s, local=%s)",
cr.remote, resp.Header.Get("X-Etcd-Cluster-ID"), cr.cid)
return nil, errClusterIDMismatch
default:
return nil, fmt.Errorf("unhandled error %q when precondition failed", string(b))
}
default:
resp.Body.Close()
return nil, fmt.Errorf("unhandled http status %d", resp.StatusCode)
}
}
// post POSTs a data payload to a url. Returns nil if the POST succeeds,
// error on any failure.
func (p *pipeline) post(data []byte) (err error) {
u := p.picker.pick()
uu := u
uu.Path = RaftPrefix
req, err := http.NewRequest("POST", uu.String(), bytes.NewBuffer(data))
if err != nil {
p.picker.unreachable(u)
return err
}
req.Header.Set("Content-Type", "application/protobuf")
req.Header.Set("X-Server-From", p.from.String())
req.Header.Set("X-Server-Version", version.Version)
req.Header.Set("X-Min-Cluster-Version", version.MinClusterVersion)
req.Header.Set("X-Etcd-Cluster-ID", p.cid.String())
var stopped bool
defer func() {
if stopped {
// rewrite to errStopped so the caller goroutine can stop itself
err = errStopped
}
}()
done := make(chan struct{}, 1)
cancel := httputil.RequestCanceler(p.tr, req)
go func() {
select {
case <-done:
case <-p.stopc:
waitSchedule()
stopped = true
cancel()
}
}()
resp, err := p.tr.RoundTrip(req)
done <- struct{}{}
if err != nil {
p.picker.unreachable(u)
return err
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
p.picker.unreachable(u)
return err
}
resp.Body.Close()
switch resp.StatusCode {
case http.StatusPreconditionFailed:
switch strings.TrimSuffix(string(b), "\n") {
case errIncompatibleVersion.Error():
plog.Errorf("request sent was ignored by peer %s (server version incompatible)", p.to)
return errIncompatibleVersion
case errClusterIDMismatch.Error():
plog.Errorf("request sent was ignored (cluster ID mismatch: remote[%s]=%s, local=%s)",
p.to, resp.Header.Get("X-Etcd-Cluster-ID"), p.cid)
return errClusterIDMismatch
default:
return fmt.Errorf("unhandled error %q when precondition failed", string(b))
}
case http.StatusForbidden:
err := fmt.Errorf("the member has been permanently removed from the cluster")
select {
case p.errorc <- err:
default:
}
return nil
case http.StatusNoContent:
return nil
default:
return fmt.Errorf("unexpected http status %s while posting to %q", http.StatusText(resp.StatusCode), req.URL.String())
}
}
func (p *reverseProxy) ServeHTTP(rw http.ResponseWriter, clientreq *http.Request) {
proxyreq := new(http.Request)
*proxyreq = *clientreq
startTime := time.Now()
var (
proxybody []byte
err error
)
if clientreq.Body != nil {
proxybody, err = ioutil.ReadAll(clientreq.Body)
if err != nil {
msg := fmt.Sprintf("proxy: failed to read request body: %v", err)
e := httptypes.NewHTTPError(http.StatusInternalServerError, msg)
if we := e.WriteTo(rw); we != nil {
plog.Debugf("error writing HTTPError (%v) to %s", we, clientreq.RemoteAddr)
}
return
}
}
// deep-copy the headers, as these will be modified below
proxyreq.Header = make(http.Header)
copyHeader(proxyreq.Header, clientreq.Header)
normalizeRequest(proxyreq)
removeSingleHopHeaders(&proxyreq.Header)
maybeSetForwardedFor(proxyreq)
endpoints := p.director.endpoints()
if len(endpoints) == 0 {
msg := "proxy: zero endpoints currently available"
reportRequestDropped(clientreq, zeroEndpoints)
// TODO: limit the rate of the error logging.
log.Printf(msg)
e := httptypes.NewHTTPError(http.StatusServiceUnavailable, msg)
if we := e.WriteTo(rw); we != nil {
plog.Debugf("error writing HTTPError (%v) to %s", we, clientreq.RemoteAddr)
}
return
}
var requestClosed int32
completeCh := make(chan bool, 1)
closeNotifier, ok := rw.(http.CloseNotifier)
cancel := httputil.RequestCanceler(p.transport, proxyreq)
if ok {
go func() {
select {
case <-closeNotifier.CloseNotify():
atomic.StoreInt32(&requestClosed, 1)
log.Printf("proxy: client %v closed request prematurely", clientreq.RemoteAddr)
cancel()
case <-completeCh:
}
}()
defer func() {
completeCh <- true
}()
}
var res *http.Response
for _, ep := range endpoints {
if proxybody != nil {
proxyreq.Body = ioutil.NopCloser(bytes.NewBuffer(proxybody))
}
redirectRequest(proxyreq, ep.URL)
res, err = p.transport.RoundTrip(proxyreq)
if atomic.LoadInt32(&requestClosed) == 1 {
return
}
if err != nil {
reportRequestDropped(clientreq, failedSendingRequest)
log.Printf("proxy: failed to direct request to %s: %v", ep.URL.String(), err)
ep.Failed()
continue
}
break
}
if res == nil {
// TODO: limit the rate of the error logging.
msg := fmt.Sprintf("proxy: unable to get response from %d endpoint(s)", len(endpoints))
reportRequestDropped(clientreq, failedGettingResponse)
log.Printf(msg)
e := httptypes.NewHTTPError(http.StatusBadGateway, msg)
if we := e.WriteTo(rw); we != nil {
plog.Debugf("error writing HTTPError (%v) to %s", we, clientreq.RemoteAddr)
}
return
}
defer res.Body.Close()
reportRequestHandled(clientreq, res, startTime)
removeSingleHopHeaders(&res.Header)
copyHeader(rw.Header(), res.Header)
rw.WriteHeader(res.StatusCode)
io.Copy(rw, res.Body)
}
func (cr *streamReader) dial(t streamType) (io.ReadCloser, error) {
u := cr.picker.pick()
uu := u
uu.Path = path.Join(t.endpoint(), cr.tr.ID.String())
req, err := http.NewRequest("GET", uu.String(), nil)
if err != nil {
cr.picker.unreachable(u)
return nil, fmt.Errorf("failed to make http request to %v (%v)", u, err)
}
req.Header.Set("X-Server-From", cr.tr.ID.String())
req.Header.Set("X-Server-Version", version.Version)
req.Header.Set("X-Min-Cluster-Version", version.MinClusterVersion)
req.Header.Set("X-Etcd-Cluster-ID", cr.tr.ClusterID.String())
req.Header.Set("X-Raft-To", cr.peerID.String())
setPeerURLsHeader(req, cr.tr.URLs)
cr.mu.Lock()
select {
case <-cr.stopc:
cr.mu.Unlock()
return nil, fmt.Errorf("stream reader is stopped")
default:
}
cr.cancel = httputil.RequestCanceler(req)
cr.mu.Unlock()
resp, err := cr.tr.streamRt.RoundTrip(req)
if err != nil {
cr.picker.unreachable(u)
return nil, err
}
rv := serverVersion(resp.Header)
lv := semver.Must(semver.NewVersion(version.Version))
if compareMajorMinorVersion(rv, lv) == -1 && !checkStreamSupport(rv, t) {
httputil.GracefulClose(resp)
cr.picker.unreachable(u)
return nil, errUnsupportedStreamType
}
switch resp.StatusCode {
case http.StatusGone:
httputil.GracefulClose(resp)
cr.picker.unreachable(u)
reportCriticalError(errMemberRemoved, cr.errorc)
return nil, errMemberRemoved
case http.StatusOK:
return resp.Body, nil
case http.StatusNotFound:
httputil.GracefulClose(resp)
cr.picker.unreachable(u)
return nil, fmt.Errorf("peer %s failed to find local node %s", cr.peerID, cr.tr.ID)
case http.StatusPreconditionFailed:
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
cr.picker.unreachable(u)
return nil, err
}
httputil.GracefulClose(resp)
cr.picker.unreachable(u)
switch strings.TrimSuffix(string(b), "\n") {
case errIncompatibleVersion.Error():
plog.Errorf("request sent was ignored by peer %s (server version incompatible)", cr.peerID)
return nil, errIncompatibleVersion
case errClusterIDMismatch.Error():
plog.Errorf("request sent was ignored (cluster ID mismatch: peer[%s]=%s, local=%s)",
cr.peerID, resp.Header.Get("X-Etcd-Cluster-ID"), cr.tr.ClusterID)
return nil, errClusterIDMismatch
default:
return nil, fmt.Errorf("unhandled error %q when precondition failed", string(b))
}
default:
httputil.GracefulClose(resp)
cr.picker.unreachable(u)
return nil, fmt.Errorf("unhandled http status %d", resp.StatusCode)
}
}