// post POSTs a data payload to a url. Returns nil if the POST succeeds,
// error on any failure.
func (p *pipeline) post(data []byte) (err error) {
u := p.picker.pick()
req := createPostRequest(u, RaftPrefix, bytes.NewBuffer(data), "application/protobuf", p.from, p.cid)
var stopped bool
defer func() {
if stopped {
// rewrite to errStopped so the caller goroutine can stop itself
err = errStopped
}
}()
done := make(chan struct{}, 1)
cancel := httputil.RequestCanceler(p.tr, req)
go func() {
select {
case <-done:
case <-p.stopc:
waitSchedule()
stopped = true
cancel()
}
}()
resp, err := p.tr.RoundTrip(req)
done <- struct{}{}
if err != nil {
p.picker.unreachable(u)
return err
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
p.picker.unreachable(u)
return err
}
resp.Body.Close()
err = checkPostResponse(resp, b, req, p.to)
// errMemberRemoved is a critical error since a removed member should
// always be stopped. So we use reportCriticalError to report it to errorc.
if err == errMemberRemoved {
reportCriticalError(err, p.errorc)
return nil
}
return err
}
// post posts the given request.
// It returns nil when request is sent out and processed successfully.
func (s *snapshotSender) post(req *http.Request) (err error) {
cancel := httputil.RequestCanceler(s.tr, req)
type responseAndError struct {
resp *http.Response
body []byte
err error
}
result := make(chan responseAndError, 1)
go func() {
// TODO: cancel the request if it has waited for a long time(~5s) after
// it has write out the full request body, which helps to avoid receiver
// dies when sender is waiting for response
// TODO: the snapshot could be large and eat up all resources when writing
// it out. Send it block by block and rest some time between to give the
// time for main loop to run.
resp, err := s.tr.RoundTrip(req)
if err != nil {
result <- responseAndError{resp, nil, err}
return
}
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
result <- responseAndError{resp, body, err}
}()
select {
case <-s.stopc:
cancel()
return errStopped
case r := <-result:
if r.err != nil {
return r.err
}
return checkPostResponse(r.resp, r.body, req, s.to)
}
}
func (p *reverseProxy) ServeHTTP(rw http.ResponseWriter, clientreq *http.Request) {
proxyreq := new(http.Request)
*proxyreq = *clientreq
startTime := time.Now()
var (
proxybody []byte
err error
)
if clientreq.Body != nil {
proxybody, err = ioutil.ReadAll(clientreq.Body)
if err != nil {
msg := fmt.Sprintf("proxy: failed to read request body: %v", err)
e := httptypes.NewHTTPError(http.StatusInternalServerError, msg)
if we := e.WriteTo(rw); we != nil {
plog.Debugf("error writing HTTPError (%v) to %s", we, clientreq.RemoteAddr)
}
return
}
}
// deep-copy the headers, as these will be modified below
proxyreq.Header = make(http.Header)
copyHeader(proxyreq.Header, clientreq.Header)
normalizeRequest(proxyreq)
removeSingleHopHeaders(&proxyreq.Header)
maybeSetForwardedFor(proxyreq)
endpoints := p.director.endpoints()
if len(endpoints) == 0 {
msg := "proxy: zero endpoints currently available"
reportRequestDropped(clientreq, zeroEndpoints)
// TODO: limit the rate of the error logging.
log.Printf(msg)
e := httptypes.NewHTTPError(http.StatusServiceUnavailable, msg)
if we := e.WriteTo(rw); we != nil {
plog.Debugf("error writing HTTPError (%v) to %s", we, clientreq.RemoteAddr)
}
return
}
var requestClosed int32
completeCh := make(chan bool, 1)
closeNotifier, ok := rw.(http.CloseNotifier)
cancel := httputil.RequestCanceler(p.transport, proxyreq)
if ok {
go func() {
select {
case <-closeNotifier.CloseNotify():
atomic.StoreInt32(&requestClosed, 1)
log.Printf("proxy: client %v closed request prematurely", clientreq.RemoteAddr)
cancel()
case <-completeCh:
}
}()
defer func() {
completeCh <- true
}()
}
var res *http.Response
for _, ep := range endpoints {
if proxybody != nil {
proxyreq.Body = ioutil.NopCloser(bytes.NewBuffer(proxybody))
}
redirectRequest(proxyreq, ep.URL)
res, err = p.transport.RoundTrip(proxyreq)
if atomic.LoadInt32(&requestClosed) == 1 {
return
}
if err != nil {
reportRequestDropped(clientreq, failedSendingRequest)
log.Printf("proxy: failed to direct request to %s: %v", ep.URL.String(), err)
ep.Failed()
continue
}
break
}
if res == nil {
// TODO: limit the rate of the error logging.
msg := fmt.Sprintf("proxy: unable to get response from %d endpoint(s)", len(endpoints))
reportRequestDropped(clientreq, failedGettingResponse)
log.Printf(msg)
e := httptypes.NewHTTPError(http.StatusBadGateway, msg)
if we := e.WriteTo(rw); we != nil {
plog.Debugf("error writing HTTPError (%v) to %s", we, clientreq.RemoteAddr)
}
return
}
defer res.Body.Close()
reportRequestHandled(clientreq, res, startTime)
removeSingleHopHeaders(&res.Header)
copyHeader(rw.Header(), res.Header)
rw.WriteHeader(res.StatusCode)
io.Copy(rw, res.Body)
}
func (cr *streamReader) dial(t streamType) (io.ReadCloser, error) {
u := cr.picker.pick()
cr.mu.Lock()
term := cr.msgAppTerm
cr.mu.Unlock()
uu := u
uu.Path = path.Join(t.endpoint(), cr.local.String())
req, err := http.NewRequest("GET", uu.String(), nil)
if err != nil {
cr.picker.unreachable(u)
return nil, fmt.Errorf("failed to make http request to %s (%v)", u, err)
}
req.Header.Set("X-Server-From", cr.local.String())
req.Header.Set("X-Server-Version", version.Version)
req.Header.Set("X-Min-Cluster-Version", version.MinClusterVersion)
req.Header.Set("X-Etcd-Cluster-ID", cr.cid.String())
req.Header.Set("X-Raft-To", cr.remote.String())
if t == streamTypeMsgApp {
req.Header.Set("X-Raft-Term", strconv.FormatUint(term, 10))
}
cr.mu.Lock()
select {
case <-cr.stopc:
cr.mu.Unlock()
return nil, fmt.Errorf("stream reader is stopped")
default:
}
cr.cancel = httputil.RequestCanceler(cr.tr, req)
cr.mu.Unlock()
resp, err := cr.tr.RoundTrip(req)
if err != nil {
cr.picker.unreachable(u)
return nil, err
}
rv := serverVersion(resp.Header)
lv := semver.Must(semver.NewVersion(version.Version))
if compareMajorMinorVersion(rv, lv) == -1 && !checkStreamSupport(rv, t) {
resp.Body.Close()
return nil, errUnsupportedStreamType
}
switch resp.StatusCode {
case http.StatusGone:
resp.Body.Close()
err := fmt.Errorf("the member has been permanently removed from the cluster")
select {
case cr.errorc <- err:
default:
}
return nil, err
case http.StatusOK:
return resp.Body, nil
case http.StatusNotFound:
resp.Body.Close()
return nil, fmt.Errorf("remote member %s could not recognize local member", cr.remote)
case http.StatusPreconditionFailed:
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
cr.picker.unreachable(u)
return nil, err
}
resp.Body.Close()
switch strings.TrimSuffix(string(b), "\n") {
case errIncompatibleVersion.Error():
plog.Errorf("request sent was ignored by peer %s (server version incompatible)", cr.remote)
return nil, errIncompatibleVersion
case errClusterIDMismatch.Error():
plog.Errorf("request sent was ignored (cluster ID mismatch: remote[%s]=%s, local=%s)",
cr.remote, resp.Header.Get("X-Etcd-Cluster-ID"), cr.cid)
return nil, errClusterIDMismatch
default:
return nil, fmt.Errorf("unhandled error %q when precondition failed", string(b))
}
default:
resp.Body.Close()
return nil, fmt.Errorf("unhandled http status %d", resp.StatusCode)
}
}