// Round trips the request to one of the endpoints and returns the response
func (l *HttpLocation) RoundTrip(req Request) (*http.Response, error) {
for {
_, err := req.GetBody().Seek(0, 0)
if err != nil {
return nil, err
}
endpoint, err := l.loadBalancer.NextEndpoint(req)
if err != nil {
log.Errorf("Load Balancer failure: %s", err)
return nil, err
}
// Adds headers, changes urls
newRequest := l.rewriteRequest(req.GetHttpRequest(), endpoint)
// In case if error is not nil, we allow load balancer to choose the next endpoint
// e.g. to do request failover. Nil error means that we got proxied the request successfully.
response, err := l.proxyToEndpoint(endpoint, req, newRequest)
if l.options.ShouldFailover(req) {
continue
} else {
return response, err
}
}
log.Errorf("All endpoints failed!")
return nil, fmt.Errorf("All endpoints failed")
}
func (s *Service) addLocation(host *Host, loc *Location) error {
router, err := s.getPathRouter(host.Name)
if err != nil {
return err
}
// Create a load balancer that handles all the endpoints within the given location
rr, err := roundrobin.NewRoundRobin()
if err != nil {
return err
}
before := callback.NewBeforeChain()
after := callback.NewAfterChain()
options := httploc.Options{
Before: before,
After: after,
}
// Add rate limits
for _, rl := range loc.RateLimits {
limiter, err := s.newRateLimiter(rl)
if err == nil {
before.Add(rl.EtcdKey, limiter)
after.Add(rl.EtcdKey, limiter)
} else {
log.Errorf("Failed to create limiter: %s", before)
}
}
// Add connection limits
for _, cl := range loc.ConnLimits {
limiter, err := s.newConnLimiter(cl)
if err == nil {
before.Add(cl.EtcdKey, limiter)
after.Add(cl.EtcdKey, limiter)
} else {
log.Errorf("Failed to create limiter: %s", before)
}
}
// Create a location itself
location, err := httploc.NewLocationWithOptions(loc.Name, rr, options)
if err != nil {
return err
}
// Add the location to the router
if err := router.AddLocation(loc.Path, location); err != nil {
return err
}
// Once the location added, configure all endpoints
return s.configureLocation(host, loc)
}
func (c *Configurator) upsertLocation(host *Host, loc *Location) error {
if err := c.upsertHost(host); err != nil {
return err
}
// If location already exists, do nothing
if loc := c.a.GetHttpLocation(host.Name, loc.Id); loc != nil {
return nil
}
router := c.a.GetPathRouter(host.Name)
if router == nil {
return fmt.Errorf("Router not found for %s", host)
}
// Create a load balancer that handles all the endpoints within the given location
rr, err := roundrobin.NewRoundRobin()
if err != nil {
return err
}
// Create a location itself
location, err := httploc.NewLocation(loc.Id, rr)
if err != nil {
return err
}
// Always register a global connection watcher
location.GetObserverChain().Upsert(ConnWatch, c.connWatcher)
// Add the location to the router
if err := router.AddLocation(loc.Path, location); err != nil {
return err
}
// Add rate and connection limits
for _, rl := range loc.RateLimits {
if err := c.upsertLocationRateLimit(host, loc, rl); err != nil {
log.Errorf("Failed to add rate limit: %s", err)
}
}
for _, cl := range loc.ConnLimits {
if err := c.upsertLocationConnLimit(host, loc, cl); err != nil {
log.Errorf("Failed to add connection limit: %s", err)
}
}
// Once the location added, configure all endpoints
return c.syncLocationEndpoints(loc)
}
// Proxy the request to the given endpoint, execute observers and middlewares chains
func (l *HttpLocation) proxyToEndpoint(endpoint Endpoint, req Request, httpReq *http.Request) (*http.Response, error) {
a := &BaseAttempt{Endpoint: endpoint}
l.observerChain.ObserveRequest(req)
defer l.observerChain.ObserveResponse(req, a)
defer req.AddAttempt(a)
it := l.middlewareChain.GetIter()
defer l.unwindIter(it, req, a)
for v := it.Next(); v != nil; v = it.Next() {
a.Response, a.Error = v.ProcessRequest(req)
if a.Response != nil || a.Error != nil {
// Move the iterator forward to count it again once we unwind the chain
it.Next()
log.Errorf("Midleware intercepted request with response=%s, error=%s", a.Response.Status, a.Error)
return a.Response, a.Error
}
}
// Forward the request and mirror the response
start := l.options.TimeProvider.UtcNow()
a.Response, a.Error = l.transport.RoundTrip(httpReq)
a.Duration = l.options.TimeProvider.UtcNow().Sub(start)
return a.Response, a.Error
}
func (s *Service) processChange(change *Change) {
var err error
switch child := (change.Child).(type) {
case *Endpoint:
switch change.Action {
case "create":
err = s.addEndpoint((change.Parent).(*Upstream), child)
case "delete":
err = s.deleteEndpoint((change.Parent).(*Upstream), child)
}
case *Location:
switch change.Action {
case "create":
err = s.addLocation((change.Parent).(*Host), child)
case "delete":
err = s.deleteLocation((change.Parent).(*Host), child)
case "set":
if len(change.Keys["upstream"]) != 0 {
err = s.updateLocationUpstream((change.Parent).(*Host), child, change.Keys["upstream"])
} else {
err = fmt.Errorf("Unknown property update: %s", change)
}
}
case *Host:
switch change.Action {
case "create":
err = s.addHost(child)
case "delete":
err = s.deleteHost(child)
}
}
if err != nil {
log.Errorf("Processing change failed: %s", err)
}
}
func (c *Configurator) deleteEndpoint(upstream *Upstream, endpointId string, affectedLocations []*Location) error {
for _, l := range affectedLocations {
if err := c.syncLocationEndpoints(l); err != nil {
log.Errorf("Failed to sync %s endpoints err: %s", l, err)
}
}
return nil
}
func (c *Configurator) syncLocationEndpoints(location *Location) error {
rr := c.a.GetHttpLocationLb(location.Hostname, location.Id)
if rr == nil {
return fmt.Errorf("%s lb not found", location)
}
// First, collect and parse endpoints to add
newEndpoints := map[string]endpoint.Endpoint{}
for _, e := range location.Upstream.Endpoints {
ep, err := EndpointFromUrl(e.Url, e.Url)
if err != nil {
return fmt.Errorf("Failed to parse endpoint url: %s", e)
}
newEndpoints[e.Url] = ep
}
// Memorize what endpoints exist in load balancer at the moment
existingEndpoints := map[string]endpoint.Endpoint{}
for _, e := range rr.GetEndpoints() {
existingEndpoints[e.GetUrl().String()] = e
}
// First, add endpoints, that should be added and are not in lb
for _, e := range newEndpoints {
if _, exists := existingEndpoints[e.GetUrl().String()]; !exists {
if err := rr.AddEndpoint(e); err != nil {
log.Errorf("Failed to add %s, err: %s", e, err)
} else {
log.Infof("Added %s to %s", e, location)
}
}
}
// Second, remove endpoints that should not be there any more
for _, e := range existingEndpoints {
if _, exists := newEndpoints[e.GetUrl().String()]; !exists {
if err := rr.RemoveEndpoint(e); err != nil {
log.Errorf("Failed to remove %s, err: %s", e, err)
} else {
log.Infof("Removed %s from %s", e, location)
}
}
}
return nil
}
func (s *Service) configureLocation(host *Host, location *Location) error {
rr, err := s.getHttpLocationLb(host.Name, location.Name)
if err != nil {
return err
}
// First, collect and parse endpoints to add
endpointsToAdd := map[string]endpoint.Endpoint{}
for _, e := range location.Upstream.Endpoints {
ep, err := EndpointFromUrl(e.Name, e.Url)
if err != nil {
return fmt.Errorf("Failed to parse endpoint url: %s", e)
}
endpointsToAdd[ep.GetId()] = ep
}
// Memorize what endpoints exist in load balancer at the moment
existing := map[string]endpoint.Endpoint{}
for _, e := range rr.GetEndpoints() {
existing[e.GetId()] = e
}
// First, add endpoints, that should be added and are not in lb
for eid, e := range endpointsToAdd {
if _, exists := existing[eid]; !exists {
if err := rr.AddEndpoint(e); err != nil {
log.Errorf("Failed to add %s, err: %s", e, err)
} else {
log.Infof("Added %s", e)
}
}
}
// Second, remove endpoints that should not be there any more
for eid, e := range existing {
if _, exists := endpointsToAdd[eid]; !exists {
if err := rr.RemoveEndpoint(e); err != nil {
log.Errorf("Failed to remove %s, err: %s", e, err)
} else {
log.Infof("Removed %s", e)
}
}
}
return nil
}
func (c *Configurator) WatchChanges(changes chan interface{}) error {
for {
change := <-changes
if err := c.processChange(change); err != nil {
log.Errorf("Failed to process change %#v, err: %s", change, err)
}
}
return nil
}
func (s *Service) configureHost(host *Host) error {
for _, loc := range host.Locations {
if err := s.addLocation(host, loc); err != nil {
log.Errorf("Failed adding %s to %s, err: %s", loc, host, err)
} else {
log.Infof("Added %s to %s", loc, host)
}
}
return nil
}
func (s *Service) configureProxy() error {
hosts, err := s.backend.GetHosts()
if err != nil {
return err
}
for _, host := range hosts {
log.Infof("Configuring %s", host)
if err := s.addHost(host); err != nil {
log.Errorf("Failed adding %s, err: %s", host, err)
continue
}
if err := s.configureHost(host); err != nil {
log.Errorf("Failed configuring %s", host)
continue
}
}
return nil
}
func (f *JsonFormatter) Format(err ProxyError) (int, []byte, string) {
encodedError, e := json.Marshal(map[string]interface{}{
"error": string(err.Error()),
})
if e != nil {
log.Errorf("Failed to serialize: %s", e)
encodedError = []byte("{}")
}
return err.GetStatusCode(), encodedError, "application/json"
}
func (s *EtcdBackend) readLocation(hostname, locationId string) (*Location, error) {
locationKey := join(s.etcdKey, "hosts", hostname, "locations", locationId)
_, err := s.client.Get(locationKey, false, false)
if err != nil {
if notFound(err) {
return nil, fmt.Errorf("Location '%s' not found for Host '%s'", locationId, hostname)
}
return nil, err
}
path, ok := s.getVal(locationKey, "path")
if !ok {
return nil, fmt.Errorf("Missing location path: %s", locationKey)
}
upstreamKey, ok := s.getVal(locationKey, "upstream")
if !ok {
return nil, fmt.Errorf("Missing location upstream: %s", locationKey)
}
location := &Location{
Name: suffix(locationKey),
EtcdKey: locationKey,
Path: path,
ConnLimits: []*ConnLimit{},
RateLimits: []*RateLimit{},
}
upstream, err := s.readUpstream(upstreamKey)
if err != nil {
return nil, err
}
for _, e := range upstream.Endpoints {
stats, err := s.statsGetter.GetStats(hostname, locationId, e.Name)
if err == nil {
e.Stats = stats
} else {
log.Errorf("Failed to get stats about endpoint: %s, err: %s", e, err)
}
}
for _, cl := range s.getVals(locationKey, "limits", "connections") {
connLimit, err := s.readLocationConnLimit(cl.Key)
if err == nil {
location.ConnLimits = append(location.ConnLimits, connLimit)
}
}
for _, cl := range s.getVals(locationKey, "limits", "rates") {
rateLimit, err := s.readLocationRateLimit(cl.Key)
if err == nil {
location.RateLimits = append(location.RateLimits, rateLimit)
}
}
location.Upstream = upstream
return location, nil
}
func (c *Configurator) addEndpoint(upstream *Upstream, e *Endpoint, affectedLocations []*Location) error {
endpoint, err := EndpointFromUrl(e.EtcdKey, e.Url)
if err != nil {
return fmt.Errorf("Failed to parse endpoint url: %s", endpoint)
}
for _, l := range affectedLocations {
if err := c.syncLocationEndpoints(l); err != nil {
log.Errorf("Failed to sync %s endpoints err: %s", l, err)
}
}
return nil
}
func (s *EtcdBackend) watchChanges() {
waitIndex := uint64(0)
for {
response, err := s.client.Watch(s.etcdKey, waitIndex, true, nil, nil)
if err != nil {
log.Errorf("Failed to get response from etcd: %s, quitting watch goroutine", err)
return
}
log.Infof("Got response: %s %s %d %s",
response.Action, response.Node.Key, response.EtcdIndex, err)
change, err := s.parseChange(response)
if err != nil {
log.Errorf("Failed to process change: %s, ignoring", err)
continue
}
if change != nil {
s.changes <- change
}
waitIndex = response.Node.ModifiedIndex + 1
}
}
// Watches etcd changes and generates structured events telling vulcand to add or delete locations, hosts etc.
// if initialSetup is true, reads the existing configuration and generates events for inital configuration of the proxy.
func (s *EtcdBackend) WatchChanges(changes chan interface{}, initialSetup bool) error {
if initialSetup == true {
log.Infof("Etcd backend reading initial configuration")
if err := s.generateChanges(changes); err != nil {
log.Errorf("Failed to generate changes: %s, stopping watch.", err)
return err
}
}
// This index helps us to get changes in sequence, as they were performed by clients.
waitIndex := uint64(0)
for {
response, err := s.client.Watch(s.etcdKey, waitIndex, true, nil, s.cancelC)
if err != nil {
if err == etcd.ErrWatchStoppedByUser {
log.Infof("Stop watching: graceful shutdown")
return nil
}
log.Errorf("Stop watching: Etcd client error: %v", err)
return err
}
waitIndex = response.Node.ModifiedIndex + 1
log.Infof("Got response: %s %s %d %v",
response.Action, response.Node.Key, response.EtcdIndex, err)
change, err := s.parseChange(response)
if err != nil {
log.Errorf("Failed to process change: %s, ignoring", err)
continue
}
if change != nil {
log.Infof("Sending change: %s", change)
select {
case changes <- change:
case <-s.stopC:
return nil
}
}
}
return nil
}
// Accepts requests, round trips it to the endpoint and writes backe the response.
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Record the request body so we can replay it on errors.
body, err := netutils.NewBodyBuffer(r.Body)
if err != nil || body == nil {
log.Errorf("Request read error %s", err)
p.replyError(errors.FromStatus(http.StatusBadRequest), w, r)
return
}
defer body.Close()
r.Body = body
req := &request.BaseRequest{
HttpRequest: r,
Id: atomic.AddInt64(&p.lastRequestId, 1),
Body: body,
}
err = p.proxyRequest(w, req)
if err != nil {
log.Errorf("%s failed: %s", req, err)
p.replyError(err, w, r)
}
}
// Accepts requests, round trips it to the endpoint, and writes back the response.
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Record the request body so we can replay it on errors.
body, err := netutils.NewBodyBuffer(r.Body)
if err != nil || body == nil {
log.Errorf("Request read error %s", err)
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
p.replyError(errors.FromStatus(http.StatusRequestTimeout), w, r)
} else {
p.replyError(errors.FromStatus(http.StatusBadRequest), w, r)
}
return
}
defer body.Close()
r.Body = body
req := request.NewBaseRequest(r, atomic.AddInt64(&p.lastRequestId, 1), body)
err = p.proxyRequest(w, req)
if err != nil {
log.Errorf("%s failed: %s", req, err)
p.replyError(err, w, r)
}
}
func (r *RoundRobin) adjustWeights() {
if r.options.FailureHandler == nil {
return
}
weights, err := r.options.FailureHandler.AdjustWeights()
if err != nil {
log.Errorf("%s returned error: %s", r.options.FailureHandler, err)
return
}
changed := false
for _, w := range weights {
if w.GetEndpoint().GetEffectiveWeight() != w.GetWeight() {
w.GetEndpoint().setEffectiveWeight(w.GetWeight())
changed = true
}
}
if changed {
r.resetIterator()
}
}
// Round trips the request to the selected location and writes back the response
func (p *Proxy) proxyRequest(w http.ResponseWriter, req *request.BaseRequest) error {
location, err := p.router.Route(req)
if err != nil {
return err
}
// Router could not find a matching location, we can do nothing more
if location == nil {
log.Errorf("%s failed to route", req)
return errors.FromStatus(http.StatusBadGateway)
}
response, err := location.RoundTrip(req)
if response != nil {
netutils.CopyHeaders(w.Header(), response.Header)
w.WriteHeader(response.StatusCode)
io.Copy(w, response.Body)
defer response.Body.Close()
return nil
} else {
return err
}
}
func (s *Service) deleteEndpoint(upstream *Upstream, e *Endpoint) error {
endpoint, err := EndpointFromUrl(e.Name, "http://delete.me:4000")
if err != nil {
return fmt.Errorf("Failed to parse endpoint url: %s", endpoint)
}
locations, err := s.getLocations(upstream.Name)
if err != nil {
return err
}
for _, l := range locations {
rr, ok := l.GetLoadBalancer().(*roundrobin.RoundRobin)
if !ok {
return fmt.Errorf("Unexpected load balancer type: %T", l.GetLoadBalancer())
}
if err := rr.RemoveEndpoint(endpoint); err != nil {
log.Errorf("Failed to remove endpoint: %s", err)
} else {
log.Infof("Removed %s", e)
}
}
return nil
}