// Make sure we failover if one upstream is too slow to respond
func (s *ProxySuite) TestUpstreamServerTimeout(c *C) {
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(time.Second * time.Duration(100))
w.Write([]byte("Hi, I'm upstream"))
})
// Do not call Close as it will hang for 100 seconds
defer upstream.CloseClientConnections()
upstream2 := s.newServer(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hi, I'm upstream 2"))
})
defer upstream2.Close()
code := fmt.Sprintf(
`function handle(request){
return {failover: true, upstreams: ["%s", "%s"]};
}
`, upstream.URL, upstream2.URL)
timeout := time.Duration(10) * time.Millisecond
proxy := s.newProxyWithTimeouts(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider), timeout, timeout)
defer proxy.Close()
response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "hello!")
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream 2")
}
func (s *ProxySuite) TestForwardHeadersAdded(c *C) {
var capturedHeaders http.Header
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
capturedHeaders = r.Header
w.Write([]byte("Hi, I'm upstream"))
})
defer upstream.Close()
code := fmt.Sprintf(
`function handle(request){
return {failover: true, upstreams: ["%s"]};
}
`, upstream.URL)
proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
response, bodyBytes := s.Get(c,
proxy.URL,
http.Header{"Host": []string{"crazyhostname.example.com"}},
"hello!")
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream")
hostname, _ := os.Hostname()
c.Assert(capturedHeaders.Get("X-Forwarded-For"), Equals, "127.0.0.1")
c.Assert(capturedHeaders.Get("X-Forwarded-Proto"), Equals, "http")
c.Assert(capturedHeaders.Get("X-Forwarded-Host"), Equals, "crazyhostname.example.com")
c.Assert(capturedHeaders.Get("X-Forwarded-Server"), Equals, hostname)
}
// Make sure get request in proxy works
func (s *ProxySuite) TestGetRequestInProxyAuth(c *C) {
var query url.Values
var headers http.Header
control := s.newServer(func(w http.ResponseWriter, r *http.Request) {
headers = r.Header
query = r.URL.Query()
w.Write([]byte(`{"response": "hello!"}`))
})
defer control.Close()
code := fmt.Sprintf(
`function handle(request){
return get("%s", request.query, request.auth)
}
`, control.URL)
proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
response, bodyBytes := s.Get(c, fmt.Sprintf("%s?a=b&a=c&x=y", proxy.URL), s.authHeaders, "hello!")
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, `{"response":"hello!"}`)
c.Assert(query, DeepEquals, url.Values{"a": []string{"b", "c"}, "x": []string{"y"}})
c.Assert(headers.Get("Authorization"), DeepEquals, s.authHeaders.Get("Authorization"))
}
// Make sure get request in proxy works despite of the one server being down
func (s *ProxySuite) TestGetRequestInProxyFailoverOnTimeout(c *C) {
slowUpstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(time.Second * time.Duration(100))
w.Write([]byte(`{"response": "hi, I'm super slow"}`))
})
defer slowUpstream.CloseClientConnections()
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`{"response": "hi, I'm fast!"}`))
})
defer upstream.Close()
code := fmt.Sprintf(
`function handle(request){
return get(["%s", "%s"])
}
`, slowUpstream.URL, upstream.URL)
timeout := time.Duration(10) * time.Millisecond
proxy := s.newProxyWithTimeouts(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider), timeout, timeout)
defer proxy.Close()
response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "hello!")
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, `{"response":"hi, I'm fast!"}`)
}
// Make sure upstream headers were removed from the request
func (s *ProxySuite) TestHeadersRemoved(c *C) {
var customHeaders http.Header
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
customHeaders = r.Header
w.Write([]byte("Hi, I'm upstream"))
})
defer upstream.Close()
code := fmt.Sprintf(
`function handle(request){
return {
upstreams: ["%s"],
remove_headers: ["x-authorized", "X-Account-id"],
};
}
`, upstream.URL)
proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
headers := make(http.Header)
headers.Add("X-Authorized", "yes")
headers.Add("X-Authorized", "sure")
headers.Add("X-Account-Id", "a")
response, bodyBytes := s.Get(c, proxy.URL, headers, "hello!")
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream")
// make sure the headers are removed
for key, _ := range headers {
c.Assert(customHeaders.Get(key), Equals, "")
}
}
// Make sure upstream headers were added to the request
func (s *ProxySuite) TestHeadersAdded(c *C) {
var customHeaders http.Header
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
customHeaders = r.Header
w.Write([]byte("Hi, I'm upstream"))
})
defer upstream.Close()
code := fmt.Sprintf(
`function handle(request){
return {
upstreams: ["%s"],
add_headers: {"X-Header-A": ["val"], "X-Header-B": ["val2"]},
};
}
`, upstream.URL)
proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "hello!")
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream")
// make sure the headers are set
c.Assert(customHeaders["X-Header-A"][0], Equals, "val")
c.Assert(customHeaders["X-Header-B"][0], Equals, "val2")
}
// One of the upstreams consumed the request, but freezed and does not respond
func (s *ProxySuite) TestFailedUpstreamPostTimeoutFailover(c *C) {
var postedValues url.Values
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
c.Assert(r.ParseForm(), IsNil)
postedValues = r.PostForm
w.Write([]byte("Hi, I'm fast upstream!"))
})
defer upstream.Close()
slowUpstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
time.Sleep(time.Second * time.Duration(100))
w.Write([]byte("Hi, I'm slow upstream!"))
})
defer slowUpstream.CloseClientConnections()
code := fmt.Sprintf(
`function handle(request){
return {failover: true, upstreams: ["%s", "%s"]};
}
`, slowUpstream.URL, upstream.URL)
timeout := time.Duration(10) * time.Millisecond
proxy := s.newProxyWithTimeouts(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider), timeout, timeout)
defer proxy.Close()
response, bodyBytes := s.Post(c, proxy.URL, s.authHeaders, url.Values{"key": {"Value"}, "id": {"123"}})
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, "Hi, I'm fast upstream!")
c.Assert(postedValues.Get("key"), Equals, "Value")
c.Assert(postedValues.Get("id"), Equals, "123")
}
// Make sure we don't panic when all upstreams are down
func (s *ProxySuite) TestUpstreamUpstreamIsDown(c *C) {
code := `function handle(request){
return {upstreams: ["http://localhost:9999"]};
}`
proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider))
response, _ := s.Get(c, proxy.URL, s.authHeaders, "")
c.Assert(response.StatusCode, Equals, http.StatusBadGateway)
}
func (s *Service) addLocation(host *Host, loc *Location) error {
router, err := s.getPathRouter(host.Name)
if err != nil {
return err
}
// Create a load balancer that handles all the endpoints within the given location
rr, err := roundrobin.NewRoundRobin()
if err != nil {
return err
}
before := callback.NewBeforeChain()
after := callback.NewAfterChain()
options := httploc.Options{
Before: before,
After: after,
}
// Add rate limits
for _, rl := range loc.RateLimits {
limiter, err := s.newRateLimiter(rl)
if err == nil {
before.Add(rl.EtcdKey, limiter)
after.Add(rl.EtcdKey, limiter)
} else {
log.Errorf("Failed to create limiter: %s", before)
}
}
// Add connection limits
for _, cl := range loc.ConnLimits {
limiter, err := s.newConnLimiter(cl)
if err == nil {
before.Add(cl.EtcdKey, limiter)
after.Add(cl.EtcdKey, limiter)
} else {
log.Errorf("Failed to create limiter: %s", before)
}
}
// Create a location itself
location, err := httploc.NewLocationWithOptions(loc.Name, rr, options)
if err != nil {
return err
}
// Add the location to the router
if err := router.AddLocation(loc.Path, location); err != nil {
return err
}
// Once the location added, configure all endpoints
return s.configureLocation(host, loc)
}
func (c *Configurator) upsertLocation(host *Host, loc *Location) error {
if err := c.upsertHost(host); err != nil {
return err
}
// If location already exists, do nothing
if loc := c.a.GetHttpLocation(host.Name, loc.Id); loc != nil {
return nil
}
router := c.a.GetPathRouter(host.Name)
if router == nil {
return fmt.Errorf("Router not found for %s", host)
}
// Create a load balancer that handles all the endpoints within the given location
rr, err := roundrobin.NewRoundRobin()
if err != nil {
return err
}
// Create a location itself
location, err := httploc.NewLocation(loc.Id, rr)
if err != nil {
return err
}
// Always register a global connection watcher
location.GetObserverChain().Upsert(ConnWatch, c.connWatcher)
// Add the location to the router
if err := router.AddLocation(loc.Path, location); err != nil {
return err
}
// Add rate and connection limits
for _, rl := range loc.RateLimits {
if err := c.upsertLocationRateLimit(host, loc, rl); err != nil {
log.Errorf("Failed to add rate limit: %s", err)
}
}
for _, cl := range loc.ConnLimits {
if err := c.upsertLocationConnLimit(host, loc, cl); err != nil {
log.Errorf("Failed to add connection limit: %s", err)
}
}
// Once the location added, configure all endpoints
return c.syncLocationEndpoints(loc)
}
// Proxy denies request
func (s *ProxySuite) TestProxyAccessDenied(c *C) {
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("I am upstream!"))
})
defer upstream.Close()
code := `function handle(request){
return {code: 403, body: {error: "Forbidden"}};
}`
proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "")
c.Assert(response.StatusCode, Equals, http.StatusForbidden)
c.Assert(string(bodyBytes), Equals, `{"error":"Forbidden"}`)
}
// Make sure we stil forwarded the request even if the rate limiter failed
func (s *ProxySuite) TestUpstreamRateLimiterDown(c *C) {
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hi, I'm upstream!"))
})
defer upstream.Close()
code := fmt.Sprintf(
`function handle(request){
return {upstreams: ["%s"], rates: {all: "10 requests/minute"}};
}
`, upstream.URL)
proxy := s.newProxy(code, &backend.FailingBackend{}, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "")
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream!")
}
func (s *ProxySuite) TestUpstreamGetFailover(c *C) {
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hi, I'm upstream!"))
})
defer upstream.Close()
code := fmt.Sprintf(
`function handle(request){
return {failover: true, upstreams: ["%s", "%s"]};
}
`, "http://localhost:9999", upstream.URL)
proxy := s.newProxy(code, &backend.FailingBackend{}, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "")
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream!")
}
// Make sure get request in proxy works despite of the one server being down
func (s *ProxySuite) TestGetRequestInProxyFailover(c *C) {
control := s.newServer(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`{"response": "hi!"}`))
})
defer control.Close()
code := fmt.Sprintf(
`function handle(request){
return get(["http://localhost:9999", "%s"])
}
`, control.URL)
proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "hello!")
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, `{"response":"hi!"}`)
}
func (s *ProxySuite) TestProxyAuthRequired(c *C) {
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hi, I'm upstream"))
})
defer upstream.Close()
code := fmt.Sprintf(
`function handle(request){
if(!request.username || ! request.password)
return {code: 401, body: {error: "Unauthorized"}};
return {upstreams: ["%s"]};
}
`, upstream.URL)
proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
response, bodyBytes := s.Get(c, proxy.URL, http.Header{}, "")
c.Assert(response.StatusCode, Equals, http.StatusUnauthorized)
c.Assert(string(bodyBytes), Equals, fmt.Sprintf(`{"error":"%s"}`, http.StatusText(http.StatusUnauthorized)))
}
func (s *Service) addLocation(host *Host, loc *Location) error {
router, err := s.getPathRouter(host.Name)
if err != nil {
return err
}
// Create a load balancer that handles all the endpoints within the given location
rr, err := roundrobin.NewRoundRobin()
if err != nil {
return err
}
// Create a location itself
location, err := httploc.NewLocation(loc.Name, rr)
if err != nil {
return err
}
// Add the location to the router
if err := router.AddLocation(loc.Path, location); err != nil {
return err
}
// Once the location added, configure all endpoints
return s.configureLocation(host, loc)
}
// Make sure hop headers were removed
func (s *ProxySuite) TestHopHeadersRemoved(c *C) {
var capturedHeaders http.Header
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
capturedHeaders = r.Header
w.Write([]byte("Hi, I'm upstream"))
})
defer upstream.Close()
code := fmt.Sprintf(
`function handle(request){
return {failover: true, upstreams: ["%s"]};
}
`, upstream.URL)
proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
headers := make(http.Header)
headers.Add("Connection", "close")
headers.Add("Keep-Alive", "timeout=600")
headers.Add("Proxy-Authenticate", "Negotiate")
headers.Add("Proxy-Authorization", "Basic YW55IGNhcm5hbCBwbGVhcw==")
headers.Add("Authorization", "Basic YW55IGNhcm5hbCBwbGVhcw==")
headers.Add("Te", "deflate")
headers.Add("Trailer", "a")
headers.Add("Transfer-Encoding", "chunked")
headers.Add("Upgrade", "IRC/6.9")
response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "hello!")
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream")
// make sure the headers are removed
for _, h := range hopHeaders {
c.Assert(capturedHeaders.Get(h), Equals, "")
}
}
// Make sure that path has been altered
func (s *ProxySuite) TestRewritePath(c *C) {
path := ""
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
path = r.URL.Path
w.Write([]byte("Hi, I'm upstream"))
})
defer upstream.Close()
code := fmt.Sprintf(
`function handle(request){
return {failover: true, upstreams: ["%s"], rewrite_path: "/new/path"};
}
`, upstream.URL)
proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "hello!")
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream")
c.Assert(path, Equals, "/new/path")
}
func (s *ProxySuite) TestFailedUpstreamPostFailover(c *C) {
var postedValues url.Values
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
c.Assert(r.ParseForm(), IsNil)
postedValues = r.PostForm
w.Write([]byte("Hi, I'm upstream!"))
})
defer upstream.Close()
code := fmt.Sprintf(
`function handle(request){
return {failover: true, upstreams: ["%s", "%s"]};
}
`, "http://localhost:9999", upstream.URL)
proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
response, bodyBytes := s.Post(c, proxy.URL, s.authHeaders, url.Values{"key": {"Value"}, "id": {"123"}})
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream!")
c.Assert(postedValues.Get("key"), Equals, "Value")
c.Assert(postedValues.Get("id"), Equals, "123")
}
// Make sure we've returned response with valid retry-seconds
func (s *ProxySuite) TestRateLimited(c *C) {
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hi, I'm upstream"))
})
defer upstream.Close()
// Upstream is out of capacity, we should be told to be throttled
s.backend.UpdateCount("all_requests", time.Minute, 10)
code := fmt.Sprintf(
`function handle(request){
return {upstreams: ["%s"], rates: {all: "10 requests/minute"}};
}
`, upstream.URL)
proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "")
c.Assert(response.StatusCode, Equals, 429)
m := s.loadJson(bodyBytes)
c.Assert(m["retry_seconds"], Equals, float64(53))
}
func (s *ProxySuite) TestUpstreamGetFailoverCodes(c *C) {
upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusGone)
w.Write([]byte("Hi, I'm upstream, but I'm shutting down"))
})
defer upstream.Close()
upstream2 := s.newServer(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hi, I'm upstream that you need"))
})
defer upstream2.Close()
code := fmt.Sprintf(
`function handle(request){
return {failover: {codes: [410]}, upstreams: ["%s", "%s"]};
}
`, upstream.URL, upstream2.URL)
proxy := s.newProxy(code, &backend.FailingBackend{}, roundrobin.NewRoundRobin(s.timeProvider))
defer proxy.Close()
response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "")
c.Assert(response.StatusCode, Equals, http.StatusOK)
c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream that you need")
}
func (s *Service) initProxy() (*vulcan.ReverseProxy, error) {
var b backend.Backend
var err error
if s.options.backend == "cassandra" {
cassandraConfig := &backend.CassandraConfig{
Servers: s.options.cassandraServers,
Keyspace: s.options.cassandraKeyspace,
Consistency: gocql.One,
LaunchCleanup: s.options.cassandraCleanup,
CleanupTime: s.options.cassandraCleanupOptions.T,
}
b, err = backend.NewCassandraBackend(
cassandraConfig, &timeutils.RealTime{})
if err != nil {
return nil, err
}
} else if s.options.backend == "memory" {
b, err = backend.NewMemoryBackend(&timeutils.RealTime{})
if err != nil {
return nil, err
}
} else {
return nil, fmt.Errorf("Unsupported backend")
}
var loadBalancer loadbalance.Balancer
if s.options.loadBalancer == "roundrobin" || s.options.loadBalancer == "random" {
loadBalancer = roundrobin.NewRoundRobin(&timeutils.RealTime{})
} else {
return nil, fmt.Errorf("Unsupported loadbalancing algo")
}
outputs := strings.Split(s.options.metricsOutput, ",")
for _, v := range outputs {
metrics.AddOutput(v)
}
if s.options.sslCertFile != "" && s.options.sslKeyFile == "" {
return nil, fmt.Errorf("invalid configuration: -sslkey unspecified, but -sslcert was specified.")
} else if s.options.sslCertFile == "" && s.options.sslKeyFile != "" {
return nil, fmt.Errorf("invalid configuration: -sslcert unspecified, but -sslkey was specified.")
}
var discoveryService discovery.Service
if s.options.discovery != "" {
discoveryUrl := s.options.discovery
if s.options.discovery == "etcd" {
// TODO remove this compat hack?
discoveryUrl = "etcd://" + strings.Join(s.options.etcdEndpoints, ",")
discoveryService = discovery.NewEtcd(s.options.etcdEndpoints)
}
discoveryService, err = discovery.New(discoveryUrl)
if err != nil {
return nil, err
}
}
controller := &js.JsController{
CodeGetter: js.NewFileGetter(s.options.codePath),
DiscoveryService: discoveryService,
}
proxySettings := &vulcan.ProxySettings{
Controller: controller,
ThrottlerBackend: b,
LoadBalancer: loadBalancer,
}
proxy, err := vulcan.NewReverseProxy(&s.metrics, proxySettings)
if err != nil {
return nil, err
}
controller.Client = proxy
return proxy, nil
}