// Make sure we failover if one upstream is too slow to respond func (s *ProxySuite) TestUpstreamServerTimeout(c *C) { upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { time.Sleep(time.Second * time.Duration(100)) w.Write([]byte("Hi, I'm upstream")) }) // Do not call Close as it will hang for 100 seconds defer upstream.CloseClientConnections() upstream2 := s.newServer(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("Hi, I'm upstream 2")) }) defer upstream2.Close() code := fmt.Sprintf( `function handle(request){ return {failover: true, upstreams: ["%s", "%s"]}; } `, upstream.URL, upstream2.URL) timeout := time.Duration(10) * time.Millisecond proxy := s.newProxyWithTimeouts(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider), timeout, timeout) defer proxy.Close() response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "hello!") c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream 2") }
func (s *ProxySuite) TestForwardHeadersAdded(c *C) { var capturedHeaders http.Header upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { capturedHeaders = r.Header w.Write([]byte("Hi, I'm upstream")) }) defer upstream.Close() code := fmt.Sprintf( `function handle(request){ return {failover: true, upstreams: ["%s"]}; } `, upstream.URL) proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() response, bodyBytes := s.Get(c, proxy.URL, http.Header{"Host": []string{"crazyhostname.example.com"}}, "hello!") c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream") hostname, _ := os.Hostname() c.Assert(capturedHeaders.Get("X-Forwarded-For"), Equals, "127.0.0.1") c.Assert(capturedHeaders.Get("X-Forwarded-Proto"), Equals, "http") c.Assert(capturedHeaders.Get("X-Forwarded-Host"), Equals, "crazyhostname.example.com") c.Assert(capturedHeaders.Get("X-Forwarded-Server"), Equals, hostname) }
// Make sure get request in proxy works func (s *ProxySuite) TestGetRequestInProxyAuth(c *C) { var query url.Values var headers http.Header control := s.newServer(func(w http.ResponseWriter, r *http.Request) { headers = r.Header query = r.URL.Query() w.Write([]byte(`{"response": "hello!"}`)) }) defer control.Close() code := fmt.Sprintf( `function handle(request){ return get("%s", request.query, request.auth) } `, control.URL) proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() response, bodyBytes := s.Get(c, fmt.Sprintf("%s?a=b&a=c&x=y", proxy.URL), s.authHeaders, "hello!") c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, `{"response":"hello!"}`) c.Assert(query, DeepEquals, url.Values{"a": []string{"b", "c"}, "x": []string{"y"}}) c.Assert(headers.Get("Authorization"), DeepEquals, s.authHeaders.Get("Authorization")) }
// Make sure get request in proxy works despite of the one server being down func (s *ProxySuite) TestGetRequestInProxyFailoverOnTimeout(c *C) { slowUpstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { time.Sleep(time.Second * time.Duration(100)) w.Write([]byte(`{"response": "hi, I'm super slow"}`)) }) defer slowUpstream.CloseClientConnections() upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`{"response": "hi, I'm fast!"}`)) }) defer upstream.Close() code := fmt.Sprintf( `function handle(request){ return get(["%s", "%s"]) } `, slowUpstream.URL, upstream.URL) timeout := time.Duration(10) * time.Millisecond proxy := s.newProxyWithTimeouts(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider), timeout, timeout) defer proxy.Close() response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "hello!") c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, `{"response":"hi, I'm fast!"}`) }
// Make sure upstream headers were removed from the request func (s *ProxySuite) TestHeadersRemoved(c *C) { var customHeaders http.Header upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { customHeaders = r.Header w.Write([]byte("Hi, I'm upstream")) }) defer upstream.Close() code := fmt.Sprintf( `function handle(request){ return { upstreams: ["%s"], remove_headers: ["x-authorized", "X-Account-id"], }; } `, upstream.URL) proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() headers := make(http.Header) headers.Add("X-Authorized", "yes") headers.Add("X-Authorized", "sure") headers.Add("X-Account-Id", "a") response, bodyBytes := s.Get(c, proxy.URL, headers, "hello!") c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream") // make sure the headers are removed for key, _ := range headers { c.Assert(customHeaders.Get(key), Equals, "") } }
// Make sure upstream headers were added to the request func (s *ProxySuite) TestHeadersAdded(c *C) { var customHeaders http.Header upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { customHeaders = r.Header w.Write([]byte("Hi, I'm upstream")) }) defer upstream.Close() code := fmt.Sprintf( `function handle(request){ return { upstreams: ["%s"], add_headers: {"X-Header-A": ["val"], "X-Header-B": ["val2"]}, }; } `, upstream.URL) proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "hello!") c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream") // make sure the headers are set c.Assert(customHeaders["X-Header-A"][0], Equals, "val") c.Assert(customHeaders["X-Header-B"][0], Equals, "val2") }
// One of the upstreams consumed the request, but freezed and does not respond func (s *ProxySuite) TestFailedUpstreamPostTimeoutFailover(c *C) { var postedValues url.Values upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { c.Assert(r.ParseForm(), IsNil) postedValues = r.PostForm w.Write([]byte("Hi, I'm fast upstream!")) }) defer upstream.Close() slowUpstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { r.ParseForm() time.Sleep(time.Second * time.Duration(100)) w.Write([]byte("Hi, I'm slow upstream!")) }) defer slowUpstream.CloseClientConnections() code := fmt.Sprintf( `function handle(request){ return {failover: true, upstreams: ["%s", "%s"]}; } `, slowUpstream.URL, upstream.URL) timeout := time.Duration(10) * time.Millisecond proxy := s.newProxyWithTimeouts(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider), timeout, timeout) defer proxy.Close() response, bodyBytes := s.Post(c, proxy.URL, s.authHeaders, url.Values{"key": {"Value"}, "id": {"123"}}) c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, "Hi, I'm fast upstream!") c.Assert(postedValues.Get("key"), Equals, "Value") c.Assert(postedValues.Get("id"), Equals, "123") }
// Make sure we don't panic when all upstreams are down func (s *ProxySuite) TestUpstreamUpstreamIsDown(c *C) { code := `function handle(request){ return {upstreams: ["http://localhost:9999"]}; }` proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider)) response, _ := s.Get(c, proxy.URL, s.authHeaders, "") c.Assert(response.StatusCode, Equals, http.StatusBadGateway) }
func (s *Service) addLocation(host *Host, loc *Location) error { router, err := s.getPathRouter(host.Name) if err != nil { return err } // Create a load balancer that handles all the endpoints within the given location rr, err := roundrobin.NewRoundRobin() if err != nil { return err } before := callback.NewBeforeChain() after := callback.NewAfterChain() options := httploc.Options{ Before: before, After: after, } // Add rate limits for _, rl := range loc.RateLimits { limiter, err := s.newRateLimiter(rl) if err == nil { before.Add(rl.EtcdKey, limiter) after.Add(rl.EtcdKey, limiter) } else { log.Errorf("Failed to create limiter: %s", before) } } // Add connection limits for _, cl := range loc.ConnLimits { limiter, err := s.newConnLimiter(cl) if err == nil { before.Add(cl.EtcdKey, limiter) after.Add(cl.EtcdKey, limiter) } else { log.Errorf("Failed to create limiter: %s", before) } } // Create a location itself location, err := httploc.NewLocationWithOptions(loc.Name, rr, options) if err != nil { return err } // Add the location to the router if err := router.AddLocation(loc.Path, location); err != nil { return err } // Once the location added, configure all endpoints return s.configureLocation(host, loc) }
func (c *Configurator) upsertLocation(host *Host, loc *Location) error { if err := c.upsertHost(host); err != nil { return err } // If location already exists, do nothing if loc := c.a.GetHttpLocation(host.Name, loc.Id); loc != nil { return nil } router := c.a.GetPathRouter(host.Name) if router == nil { return fmt.Errorf("Router not found for %s", host) } // Create a load balancer that handles all the endpoints within the given location rr, err := roundrobin.NewRoundRobin() if err != nil { return err } // Create a location itself location, err := httploc.NewLocation(loc.Id, rr) if err != nil { return err } // Always register a global connection watcher location.GetObserverChain().Upsert(ConnWatch, c.connWatcher) // Add the location to the router if err := router.AddLocation(loc.Path, location); err != nil { return err } // Add rate and connection limits for _, rl := range loc.RateLimits { if err := c.upsertLocationRateLimit(host, loc, rl); err != nil { log.Errorf("Failed to add rate limit: %s", err) } } for _, cl := range loc.ConnLimits { if err := c.upsertLocationConnLimit(host, loc, cl); err != nil { log.Errorf("Failed to add connection limit: %s", err) } } // Once the location added, configure all endpoints return c.syncLocationEndpoints(loc) }
// Proxy denies request func (s *ProxySuite) TestProxyAccessDenied(c *C) { upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("I am upstream!")) }) defer upstream.Close() code := `function handle(request){ return {code: 403, body: {error: "Forbidden"}}; }` proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "") c.Assert(response.StatusCode, Equals, http.StatusForbidden) c.Assert(string(bodyBytes), Equals, `{"error":"Forbidden"}`) }
// Make sure we stil forwarded the request even if the rate limiter failed func (s *ProxySuite) TestUpstreamRateLimiterDown(c *C) { upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("Hi, I'm upstream!")) }) defer upstream.Close() code := fmt.Sprintf( `function handle(request){ return {upstreams: ["%s"], rates: {all: "10 requests/minute"}}; } `, upstream.URL) proxy := s.newProxy(code, &backend.FailingBackend{}, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "") c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream!") }
func (s *ProxySuite) TestUpstreamGetFailover(c *C) { upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("Hi, I'm upstream!")) }) defer upstream.Close() code := fmt.Sprintf( `function handle(request){ return {failover: true, upstreams: ["%s", "%s"]}; } `, "http://localhost:9999", upstream.URL) proxy := s.newProxy(code, &backend.FailingBackend{}, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "") c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream!") }
// Make sure get request in proxy works despite of the one server being down func (s *ProxySuite) TestGetRequestInProxyFailover(c *C) { control := s.newServer(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`{"response": "hi!"}`)) }) defer control.Close() code := fmt.Sprintf( `function handle(request){ return get(["http://localhost:9999", "%s"]) } `, control.URL) proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "hello!") c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, `{"response":"hi!"}`) }
func (s *ProxySuite) TestProxyAuthRequired(c *C) { upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("Hi, I'm upstream")) }) defer upstream.Close() code := fmt.Sprintf( `function handle(request){ if(!request.username || ! request.password) return {code: 401, body: {error: "Unauthorized"}}; return {upstreams: ["%s"]}; } `, upstream.URL) proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() response, bodyBytes := s.Get(c, proxy.URL, http.Header{}, "") c.Assert(response.StatusCode, Equals, http.StatusUnauthorized) c.Assert(string(bodyBytes), Equals, fmt.Sprintf(`{"error":"%s"}`, http.StatusText(http.StatusUnauthorized))) }
func (s *Service) addLocation(host *Host, loc *Location) error { router, err := s.getPathRouter(host.Name) if err != nil { return err } // Create a load balancer that handles all the endpoints within the given location rr, err := roundrobin.NewRoundRobin() if err != nil { return err } // Create a location itself location, err := httploc.NewLocation(loc.Name, rr) if err != nil { return err } // Add the location to the router if err := router.AddLocation(loc.Path, location); err != nil { return err } // Once the location added, configure all endpoints return s.configureLocation(host, loc) }
// Make sure hop headers were removed func (s *ProxySuite) TestHopHeadersRemoved(c *C) { var capturedHeaders http.Header upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { capturedHeaders = r.Header w.Write([]byte("Hi, I'm upstream")) }) defer upstream.Close() code := fmt.Sprintf( `function handle(request){ return {failover: true, upstreams: ["%s"]}; } `, upstream.URL) proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() headers := make(http.Header) headers.Add("Connection", "close") headers.Add("Keep-Alive", "timeout=600") headers.Add("Proxy-Authenticate", "Negotiate") headers.Add("Proxy-Authorization", "Basic YW55IGNhcm5hbCBwbGVhcw==") headers.Add("Authorization", "Basic YW55IGNhcm5hbCBwbGVhcw==") headers.Add("Te", "deflate") headers.Add("Trailer", "a") headers.Add("Transfer-Encoding", "chunked") headers.Add("Upgrade", "IRC/6.9") response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "hello!") c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream") // make sure the headers are removed for _, h := range hopHeaders { c.Assert(capturedHeaders.Get(h), Equals, "") } }
// Make sure that path has been altered func (s *ProxySuite) TestRewritePath(c *C) { path := "" upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { path = r.URL.Path w.Write([]byte("Hi, I'm upstream")) }) defer upstream.Close() code := fmt.Sprintf( `function handle(request){ return {failover: true, upstreams: ["%s"], rewrite_path: "/new/path"}; } `, upstream.URL) proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "hello!") c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream") c.Assert(path, Equals, "/new/path") }
func (s *ProxySuite) TestFailedUpstreamPostFailover(c *C) { var postedValues url.Values upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { c.Assert(r.ParseForm(), IsNil) postedValues = r.PostForm w.Write([]byte("Hi, I'm upstream!")) }) defer upstream.Close() code := fmt.Sprintf( `function handle(request){ return {failover: true, upstreams: ["%s", "%s"]}; } `, "http://localhost:9999", upstream.URL) proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() response, bodyBytes := s.Post(c, proxy.URL, s.authHeaders, url.Values{"key": {"Value"}, "id": {"123"}}) c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream!") c.Assert(postedValues.Get("key"), Equals, "Value") c.Assert(postedValues.Get("id"), Equals, "123") }
// Make sure we've returned response with valid retry-seconds func (s *ProxySuite) TestRateLimited(c *C) { upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("Hi, I'm upstream")) }) defer upstream.Close() // Upstream is out of capacity, we should be told to be throttled s.backend.UpdateCount("all_requests", time.Minute, 10) code := fmt.Sprintf( `function handle(request){ return {upstreams: ["%s"], rates: {all: "10 requests/minute"}}; } `, upstream.URL) proxy := s.newProxy(code, s.backend, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "") c.Assert(response.StatusCode, Equals, 429) m := s.loadJson(bodyBytes) c.Assert(m["retry_seconds"], Equals, float64(53)) }
func (s *ProxySuite) TestUpstreamGetFailoverCodes(c *C) { upstream := s.newServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusGone) w.Write([]byte("Hi, I'm upstream, but I'm shutting down")) }) defer upstream.Close() upstream2 := s.newServer(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("Hi, I'm upstream that you need")) }) defer upstream2.Close() code := fmt.Sprintf( `function handle(request){ return {failover: {codes: [410]}, upstreams: ["%s", "%s"]}; } `, upstream.URL, upstream2.URL) proxy := s.newProxy(code, &backend.FailingBackend{}, roundrobin.NewRoundRobin(s.timeProvider)) defer proxy.Close() response, bodyBytes := s.Get(c, proxy.URL, s.authHeaders, "") c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(string(bodyBytes), Equals, "Hi, I'm upstream that you need") }
func (s *Service) initProxy() (*vulcan.ReverseProxy, error) { var b backend.Backend var err error if s.options.backend == "cassandra" { cassandraConfig := &backend.CassandraConfig{ Servers: s.options.cassandraServers, Keyspace: s.options.cassandraKeyspace, Consistency: gocql.One, LaunchCleanup: s.options.cassandraCleanup, CleanupTime: s.options.cassandraCleanupOptions.T, } b, err = backend.NewCassandraBackend( cassandraConfig, &timeutils.RealTime{}) if err != nil { return nil, err } } else if s.options.backend == "memory" { b, err = backend.NewMemoryBackend(&timeutils.RealTime{}) if err != nil { return nil, err } } else { return nil, fmt.Errorf("Unsupported backend") } var loadBalancer loadbalance.Balancer if s.options.loadBalancer == "roundrobin" || s.options.loadBalancer == "random" { loadBalancer = roundrobin.NewRoundRobin(&timeutils.RealTime{}) } else { return nil, fmt.Errorf("Unsupported loadbalancing algo") } outputs := strings.Split(s.options.metricsOutput, ",") for _, v := range outputs { metrics.AddOutput(v) } if s.options.sslCertFile != "" && s.options.sslKeyFile == "" { return nil, fmt.Errorf("invalid configuration: -sslkey unspecified, but -sslcert was specified.") } else if s.options.sslCertFile == "" && s.options.sslKeyFile != "" { return nil, fmt.Errorf("invalid configuration: -sslcert unspecified, but -sslkey was specified.") } var discoveryService discovery.Service if s.options.discovery != "" { discoveryUrl := s.options.discovery if s.options.discovery == "etcd" { // TODO remove this compat hack? discoveryUrl = "etcd://" + strings.Join(s.options.etcdEndpoints, ",") discoveryService = discovery.NewEtcd(s.options.etcdEndpoints) } discoveryService, err = discovery.New(discoveryUrl) if err != nil { return nil, err } } controller := &js.JsController{ CodeGetter: js.NewFileGetter(s.options.codePath), DiscoveryService: discoveryService, } proxySettings := &vulcan.ProxySettings{ Controller: controller, ThrottlerBackend: b, LoadBalancer: loadBalancer, } proxy, err := vulcan.NewReverseProxy(&s.metrics, proxySettings) if err != nil { return nil, err } controller.Client = proxy return proxy, nil }