// RegisterApp adds a new backend and a single server with Vulcand.
func (s *LeaderRegistry) RegisterApp(registration *AppRegistration) error {
log.Infof("Registering app: %v", registration)
endpoint, err := vulcan.NewEndpointWithID(s.Group, registration.Name, registration.Host, registration.Port)
if err != nil {
return err
}
err = s.client.RegisterBackend(endpoint)
if err != nil {
log.Errorf("Failed to register backend for endpoint: %v, %s", endpoint, err)
return err
}
if s.IsMaster {
err = s.maintainLeader(endpoint)
} else {
err = s.initLeader(endpoint)
}
if err != nil {
log.Errorf("Failed to register server for endpoint: %v, %s", endpoint, err)
return err
}
return nil
}
// Subscribe watches etcd changes and generates structured events telling vulcand to add or delete frontends, hosts etc.
// It is a blocking function.
func (n *ng) Subscribe(changes chan interface{}, cancelC chan bool) error {
// This index helps us to get changes in sequence, as they were performed by clients.
waitIndex := uint64(0)
for {
response, err := n.client.Watch(n.etcdKey, waitIndex, true, nil, cancelC)
if err != nil {
switch err {
case etcd.ErrWatchStoppedByUser:
log.Infof("Stop watching: graceful shutdown")
return nil
default:
log.Errorf("unexpected error: %s, stop watching", err)
return err
}
}
waitIndex = response.EtcdIndex + 1
log.Infof("%s", responseToString(response))
change, err := n.parseChange(response)
if err != nil {
log.Warningf("Ignore '%s', error: %s", responseToString(response), err)
continue
}
if change != nil {
log.Infof("%v", change)
select {
case changes <- change:
case <-cancelC:
return nil
}
}
}
}
// Subscribe watches etcd changes and generates structured events telling vulcand to add or delete frontends, hosts etc.
// It is a blocking function.
func (n *ng) Subscribe(changes chan interface{}, cancelC chan bool) error {
w := n.kapi.Watcher(n.etcdKey, &etcd.WatcherOptions{AfterIndex: 0, Recursive: true})
for {
response, err := w.Next(n.context)
if err != nil {
switch err {
case context.Canceled:
log.Infof("Stop watching: graceful shutdown")
return nil
default:
log.Errorf("unexpected error: %s, stop watching", err)
return err
}
}
log.Infof("%s", responseToString(response))
change, err := n.parseChange(response)
if err != nil {
log.Warningf("Ignore '%s', error: %s", responseToString(response), err)
continue
}
if change != nil {
log.Infof("%v", change)
select {
case changes <- change:
case <-cancelC:
return nil
}
}
}
}
// registerLocationForScope registers a location with a specified scope.
func (app *App) registerLocationForScope(methods []string, path string, scope Scope, middlewares []middleware.Middleware) {
host, err := app.apiHostForScope(scope)
if err != nil {
log.Errorf("Failed to register a location: %v", err)
return
}
app.registerLocationForHost(methods, path, host, middlewares)
}
// RegisterHandler registers the frontends and middlewares with Vulcand.
func (s *LeaderRegistry) RegisterHandler(registration *HandlerRegistration) error {
log.Infof("Registering handler: %v", registration)
location := vulcan.NewLocation(registration.Host, registration.Methods, registration.Path, registration.Name, registration.Middlewares)
err := s.client.RegisterFrontend(location)
if err != nil {
log.Errorf("Failed to register frontend for location: %v, %s", location, err)
return err
}
err = s.client.RegisterMiddleware(location)
if err != nil {
log.Errorf("Failed to register middleware for location: %v, %s", location, err)
return err
}
return nil
}
func (n *ng) newHttpTransport() etcd.CancelableTransport {
var cc *tls.Config = nil
if n.options.EtcdCertFile != "" && n.options.EtcdKeyFile != "" {
var rpool *x509.CertPool = nil
if n.options.EtcdCaFile != "" {
if pemBytes, err := ioutil.ReadFile(n.options.EtcdCaFile); err == nil {
rpool = x509.NewCertPool()
rpool.AppendCertsFromPEM(pemBytes)
} else {
log.Errorf("Error reading Etcd Cert CA File: %v", err)
}
}
if tlsCert, err := tls.LoadX509KeyPair(n.options.EtcdCertFile, n.options.EtcdKeyFile); err == nil {
cc = &tls.Config{
RootCAs: rpool,
Certificates: []tls.Certificate{tlsCert},
InsecureSkipVerify: true,
}
} else {
log.Errorf("Error loading KeyPair for TLS client: %v", err)
}
}
//Copied from etcd.DefaultTransport declaration
//Wasn't sure how to make a clean reliable deep-copy, and instead
//creating a new object was the safest and most reliable assurance
//that we aren't overwriting some global struct potentially
//shared by other etcd users.
tr := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: cc,
}
return tr
}
func (rw *rewriteHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
oldURL := rawURL(req)
// only continue if the Regexp param matches the URL
if !rw.regexp.MatchString(oldURL) {
rw.next.ServeHTTP(w, req)
return
}
// apply a rewrite regexp to the URL
newURL := rw.regexp.ReplaceAllString(oldURL, rw.replacement)
// replace any variables that may be in there
rewrittenURL := &bytes.Buffer{}
if err := ApplyString(newURL, rewrittenURL, req); err != nil {
rw.errHandler.ServeHTTP(w, req, err)
return
}
// parse the rewritten URL and replace request URL with it
parsedURL, err := url.Parse(rewrittenURL.String())
if err != nil {
rw.errHandler.ServeHTTP(w, req, err)
return
}
if rw.redirect && newURL != oldURL {
(&redirectHandler{u: parsedURL}).ServeHTTP(w, req)
return
}
req.URL = parsedURL
// make sure the request URI corresponds the rewritten URL
req.RequestURI = req.URL.RequestURI()
if !rw.rewriteBody {
rw.next.ServeHTTP(w, req)
return
}
bw := &bufferWriter{header: make(http.Header), buffer: &bytes.Buffer{}}
newBody := &bytes.Buffer{}
rw.next.ServeHTTP(bw, req)
if err := Apply(bw.buffer, newBody, req); err != nil {
log.Errorf("Failed to rewrite response body: %v", err)
return
}
utils.CopyHeaders(w.Header(), bw.Header())
w.Header().Set("Content-Length", strconv.Itoa(newBody.Len()))
w.WriteHeader(bw.code)
io.Copy(w, newBody)
}
// syncs backend servers and rebalancer state
func syncServers(m *mux, rb *roundrobin.Rebalancer, backend *backend, w *RTWatcher) error {
// First, collect and parse servers to add
newServers := map[string]*url.URL{}
for _, s := range backend.servers {
u, err := url.Parse(s.URL)
if err != nil {
return fmt.Errorf("failed to parse url %v", s.URL)
}
newServers[s.URL] = u
}
// Memorize what endpoints exist in load balancer at the moment
existingServers := map[string]*url.URL{}
for _, s := range rb.Servers() {
existingServers[s.String()] = s
}
// First, add endpoints, that should be added and are not in lb
for _, s := range newServers {
if _, exists := existingServers[s.String()]; !exists {
if err := rb.UpsertServer(s); err != nil {
log.Errorf("%v failed to add %v, err: %s", m, s, err)
} else {
log.Infof("%v add %v", m, s)
}
w.upsertServer(s)
}
}
// Second, remove endpoints that should not be there any more
for k, v := range existingServers {
if _, exists := newServers[k]; !exists {
if err := rb.RemoveServer(v); err != nil {
log.Errorf("%v failed to remove %v, err: %v", m, v, err)
} else {
log.Infof("%v removed %v", m, v)
}
w.removeServer(v)
}
}
return nil
}
func (s *stapler) updateStaple(e *stapleFetched) bool {
s.mtx.Lock()
defer s.mtx.Unlock()
hs, ok := s.v[e.hostName]
if !ok || hs.id != e.id {
log.Infof("%v: %v replaced or removed", s, hs)
// the stapler may have been replaced by concurrent call to StapleHost()
// we are going to discard this stapler and it's event as it's irrelevant
return false
}
if e.err != nil {
log.Errorf("%v failed to fetch staple response for %v, error: %v", s, hs, e.err)
if hs.response.Response.NextUpdate.Before(hs.s.clock.UtcNow()) {
log.Errorf("%v Now: %v, next: %v retry attempts exceeded, invalidating staple %v",
s, hs.s.clock.UtcNow(), hs.response.Response.NextUpdate, hs)
delete(s.v, e.hostName)
return true
}
hs.schedule(hs.s.clock.UtcNow().Add(ErrRetryPeriod))
return false
}
hs.response = e.re
switch e.re.Response.Status {
case ocsp.Good:
log.Infof("%v got good status for %v", s, hs)
hs.schedule(hs.userUpdate(e.re.Response.NextUpdate))
case ocsp.Revoked:
// no need to reschedule if it's revoked
log.Warningf("%v revoked %v", s, hs)
case ocsp.Unknown, ocsp.ServerFailed:
log.Warningf("%v status: %v for %v", s, e.re.Response.Status, hs)
hs.schedule(hs.s.clock.UtcNow().Add(hs.period))
}
return true
}
// RegisterApp adds a new backend and a single server with Vulcand.
func (s *LBRegistry) RegisterApp(registration *AppRegistration) error {
log.Infof("Registering app: %v", registration)
endpoint, err := vulcan.NewEndpoint(registration.Name, registration.Host, registration.Port)
if err != nil {
return err
}
err = s.client.RegisterBackend(endpoint)
if err != nil {
log.Errorf("Failed to register backend for endpoint: %v, %s", endpoint, err)
return err
}
err = s.client.UpsertServer(endpoint, s.TTL)
if err != nil {
log.Errorf("Failed to register server for endpoint: %v, %s", endpoint, err)
return err
}
return nil
}
func Run(registry *plugin.Registry) error {
options, err := ParseCommandLine()
if err != nil {
return fmt.Errorf("failed to parse command line: %s", err)
}
service := NewService(options, registry)
if err := service.Start(); err != nil {
log.Errorf("Failed to start service: %v", err)
return fmt.Errorf("service start failure: %s", err)
} else {
log.Infof("Service exited gracefully")
}
return nil
}
func (mx *mux) emitMetrics() error {
c := mx.options.MetricsClient
// Emit connection stats
counts := mx.connTracker.counts()
for state, values := range counts {
for addr, count := range values {
c.Gauge(c.Metric("conns", addr, state.String()), count, 1)
}
}
// Emit frontend metrics stats
frontends, err := mx.topFrontends(nil)
if err != nil {
log.Errorf("failed to get top frontends: %v", err)
return err
}
for _, f := range frontends {
m := c.Metric("frontend", strings.Replace(f.Id, ".", "_", -1))
s := f.Stats
for _, scode := range s.Counters.StatusCodes {
// response codes counters
c.Gauge(m.Metric("code", strconv.Itoa(scode.Code)), scode.Count, 1)
}
// network errors
c.Gauge(m.Metric("neterr"), s.Counters.NetErrors, 1)
// requests
c.Gauge(m.Metric("reqs"), s.Counters.Total, 1)
// round trip times in microsecond resolution
for _, b := range s.LatencyBrackets {
c.Gauge(m.Metric("rtt", strconv.Itoa(int(b.Quantile*10.0))), int64(b.Value/time.Microsecond), 1)
}
}
return nil
}
func (st *stapler) getStaple(s *engine.HostSettings) (*StapleResponse, error) {
kp := s.KeyPair
cert, err := tls.X509KeyPair(kp.Cert, kp.Key)
if err != nil {
return nil, err
}
if len(cert.Certificate) < 2 {
return nil, fmt.Errorf("Need at least leaf and peer certificate")
}
xc, err := x509.ParseCertificate(cert.Certificate[0])
if err != nil {
return nil, err
}
xi, err := x509.ParseCertificate(cert.Certificate[1])
if err != nil {
return nil, err
}
data, err := ocsp.CreateRequest(xc, xi, &ocsp.RequestOptions{})
if err != nil {
return nil, err
}
servers := xc.OCSPServer
if len(s.OCSP.Responders) != 0 {
servers = s.OCSP.Responders
}
if len(servers) == 0 {
return nil, fmt.Errorf("No OCSP responders specified")
}
var re *ocsp.Response
var raw []byte
for _, srv := range servers {
log.Infof("OCSP about to query: %v for OCSP", srv)
issuer := xi
if s.OCSP.SkipSignatureCheck {
log.Warningf("Bypassing signature check")
// this will bypass signature check
issuer = nil
}
re, raw, err = st.getOCSPResponse(srv, data, issuer)
if err != nil {
log.Errorf("Failed to get OCSP response: %v", err)
continue
}
// it's either server failed or
if re.Status != ocsp.Good && re.Status != ocsp.Revoked {
log.Warningf("Got unsatisfactiory response: %v, try next server", re.Status)
continue
}
break
}
if err != nil {
log.Infof("OCSP fetch error: %v", err)
return nil, err
}
log.Infof("OCSP Status: %v, this update: %v, next update: %v", re.Status, re.ThisUpdate, re.NextUpdate)
return &StapleResponse{Response: re, Staple: raw}, nil
}
// Helper that replies with the 500 code and happened error message.
func ReplyInternalError(w http.ResponseWriter, message string) {
log.Errorf("Internal server error: %v", message)
Reply(w, Response{"message": message}, http.StatusInternalServerError)
}
func (s *Supervisor) init() error {
proxy, err := s.newProxy(s.lastId)
s.lastId += 1
if err != nil {
return err
}
stopNewProxy := true
defer func() {
if stopNewProxy {
proxy.Stop(true)
}
}()
if err := initProxy(s.engine, proxy); err != nil {
return err
}
// This is the first start, pass the files that could have been passed
// to us by the parent process
if s.lastId == 1 && len(s.options.Files) != 0 {
log.Infof("Passing files %v to %v", s.options.Files, proxy)
if err := proxy.TakeFiles(s.options.Files); err != nil {
return err
}
}
log.Infof("%v init() initial setup done", proxy)
oldProxy := s.getCurrentProxy()
if oldProxy != nil {
files, err := oldProxy.GetFiles()
if err != nil {
return err
}
log.Infof("%v taking files from %v to %v", s, oldProxy, proxy)
if err := proxy.TakeFiles(files); err != nil {
return err
}
}
if err := proxy.Start(); err != nil {
return err
}
if oldProxy != nil {
s.wg.Add(1)
go func() {
defer s.wg.Done()
oldProxy.Stop(true)
}()
}
// Watch and configure this instance of server
stopNewProxy = false
s.setCurrentProxy(proxy)
changesC := make(chan interface{})
// This goroutine will connect to the backend and emit the changes to the changesC channel.
// In case of any error it notifies supervisor of the error by sending an error to the channel triggering reload.
go func() {
cancelC := make(chan bool)
if err := s.engine.Subscribe(changesC, cancelC); err != nil {
log.Infof("%v engine watcher got error: '%v' will restart", proxy, err)
close(cancelC)
close(changesC)
s.restartC <- err
} else {
close(cancelC)
// Graceful shutdown without restart
log.Infof("%v engine watcher got nil error, gracefully shutdown", proxy)
s.broadcastCloseC <- true
}
}()
// This goroutine will listen for changes arriving to the changes channel and reconfigure the given server
go func() {
for {
change := <-changesC
if change == nil {
log.Infof("Stop watching changes for %s", proxy)
return
}
if err := processChange(proxy, change); err != nil {
log.Errorf("failed to process change %#v, err: %s", change, err)
}
}
}()
return nil
}