func NewFilter(config *Config) (filters.Filter, error) {
ss := &Servers{
servers: make([]Server, 0),
sshClients: lrucache.NewLRUCache(uint(len(config.Servers))),
}
for _, s := range config.Servers {
server := Server{
Address: s.Addr,
ClientConfig: &ssh.ClientConfig{
User: s.Username,
Auth: []ssh.AuthMethod{
ssh.Password(s.Password),
},
},
}
ss.servers = append(ss.servers, server)
}
tr := &http.Transport{
Dial: ss.Dial,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: false,
ClientSessionCache: tls.NewLRUClientSessionCache(1000),
},
TLSHandshakeTimeout: time.Duration(config.Transport.TLSHandshakeTimeout) * time.Second,
MaxIdleConnsPerHost: config.Transport.MaxIdleConnsPerHost,
}
return &Filter{
Config: *config,
Transport: tr,
}, nil
}
func NewETCDMetrics(logger lager.Logger, etcdOptions *ETCDOptions) (*ETCDMetrics, error) {
var tlsConfig *tls.Config
if etcdOptions.CertFile != "" && etcdOptions.KeyFile != "" {
var err error
tlsConfig, err = cfhttp.NewTLSConfig(etcdOptions.CertFile, etcdOptions.KeyFile, etcdOptions.CAFile)
if err != nil {
return nil, err
}
tlsConfig.ClientSessionCache = tls.NewLRUClientSessionCache(etcdOptions.ClientSessionCacheSize)
}
client := cfhttp.NewClient()
client.CheckRedirect = func(*http.Request, []*http.Request) error {
return errRedirected
}
if tr, ok := client.Transport.(*http.Transport); ok {
tr.TLSClientConfig = tlsConfig
} else {
return nil, errors.New("Invalid transport")
}
return &ETCDMetrics{
logger: logger,
etcdCluster: etcdOptions.ClusterUrls,
client: client,
}, nil
}
func newSecureClient(url, caFile, certFile, keyFile string, clientSessionCacheSize, maxIdleConnsPerHost int, skipVerify bool) (InternalClient, error) {
client := newClient(url)
tlsConfig, err := cfhttp.NewTLSConfig(certFile, keyFile, caFile)
if err != nil {
return nil, err
}
tlsConfig.ClientSessionCache = tls.NewLRUClientSessionCache(clientSessionCacheSize)
tlsConfig.InsecureSkipVerify = skipVerify
if tr, ok := client.httpClient.Transport.(*http.Transport); ok {
tr.TLSClientConfig = tlsConfig
tr.MaxIdleConnsPerHost = maxIdleConnsPerHost
} else {
return nil, errors.New("Invalid transport")
}
if tr, ok := client.streamingHTTPClient.Transport.(*http.Transport); ok {
tr.TLSClientConfig = tlsConfig
tr.MaxIdleConnsPerHost = maxIdleConnsPerHost
} else {
return nil, errors.New("Invalid transport")
}
return client, nil
}
// tlsConfig builds a tls.Config for dialing the upstream host. Constructed
// tls.Configs are cached on a per-masquerade basis to enable client session
// caching and reduce the amount of PEM certificate parsing.
func (serverInfo *ServerInfo) tlsConfig(masquerade *Masquerade) *tls.Config {
serverInfo.tlsConfigsMutex.Lock()
defer serverInfo.tlsConfigsMutex.Unlock()
if serverInfo.tlsConfigs == nil {
serverInfo.tlsConfigs = make(map[string]*tls.Config)
}
configKey := ""
serverName := serverInfo.Host
if masquerade != nil {
configKey = masquerade.Domain + "|" + masquerade.RootCA
serverName = masquerade.Domain
}
tlsConfig := serverInfo.tlsConfigs[configKey]
if tlsConfig == nil {
tlsConfig = &tls.Config{
ClientSessionCache: tls.NewLRUClientSessionCache(1000),
InsecureSkipVerify: serverInfo.InsecureSkipVerify,
ServerName: serverName,
}
if masquerade != nil && masquerade.RootCA != "" {
caCert, err := keyman.LoadCertificateFromPEMBytes([]byte(masquerade.RootCA))
if err != nil {
log.Fatalf("Unable to load root ca cert: %s", err)
}
tlsConfig.RootCAs = caCert.PoolContainingCert()
}
serverInfo.tlsConfigs[configKey] = tlsConfig
}
return tlsConfig
}
// SessionResumeScan tests that host is able to resume sessions across all addresses.
func sessionResumeScan(addr, hostname string) (grade Grade, output Output, err error) {
config := defaultTLSConfig(hostname)
config.ClientSessionCache = tls.NewLRUClientSessionCache(1)
conn, err := tls.DialWithDialer(Dialer, Network, addr, config)
if err != nil {
return
}
if err = conn.Close(); err != nil {
return
}
return multiscan(addr, func(addrport string) (g Grade, o Output, e error) {
var conn *tls.Conn
if conn, e = tls.DialWithDialer(Dialer, Network, addrport, config); e != nil {
return
}
conn.Close()
if o = conn.ConnectionState().DidResume; o.(bool) {
g = Good
}
return
})
}
// Dialer creates a *balancer.Dialer backed by a chained server.
func (s *ChainedServerInfo) Dialer() (*balancer.Dialer, error) {
netd := &net.Dialer{Timeout: chainedDialTimeout}
var dial func() (net.Conn, error)
if s.Cert == "" {
log.Error("No Cert configured for chained server, will dial with plain tcp")
dial = func() (net.Conn, error) {
return netd.Dial("tcp", s.Addr)
}
} else {
log.Trace("Cert configured for chained server, will dial with tls over tcp")
cert, err := keyman.LoadCertificateFromPEMBytes([]byte(s.Cert))
if err != nil {
return nil, fmt.Errorf("Unable to parse certificate: %s", err)
}
x509cert := cert.X509()
sessionCache := tls.NewLRUClientSessionCache(1000)
dial = func() (net.Conn, error) {
conn, err := tlsdialer.DialWithDialer(netd, "tcp", s.Addr, false, &tls.Config{
ClientSessionCache: sessionCache,
InsecureSkipVerify: true,
})
if err != nil {
return nil, err
}
if !conn.ConnectionState().PeerCertificates[0].Equal(x509cert) {
conn.Close()
return nil, fmt.Errorf("Server's certificate didn't match expected!")
}
return conn, err
}
}
ccfg := chained.Config{
DialServer: dial,
}
if s.AuthToken != "" {
ccfg.OnRequest = func(req *http.Request) {
req.Header.Set("X-LANTERN-AUTH-TOKEN", s.AuthToken)
}
}
d := chained.NewDialer(ccfg)
// Is this a trusted proxy that we could use for HTTP traffic?
var trusted string
if s.Trusted {
trusted = "(trusted) "
}
return &balancer.Dialer{
Label: fmt.Sprintf("%schained proxy at %s", trusted, s.Addr),
Weight: s.Weight,
QOS: s.QOS,
Trusted: s.Trusted,
Dial: func(network, addr string) (net.Conn, error) {
return withStats(d.Dial(network, addr))
},
}, nil
}
func (l *TLSLog) initConfig(config *tls.Config) {
l.log = ""
l.logRand = newLogRand(config.Rand)
config.Rand = l.logRand
config.SessionTicketsDisabled = false
if config.ClientSessionCache == nil {
config.ClientSessionCache = tls.NewLRUClientSessionCache(1)
}
}
func runHTTPServer() {
cryptoConfig := &CryptoConfig{
PKFile: "proxypk.pem",
CertFile: "proxycert.pem",
ServerTLSConfig: &tls.Config{
CipherSuites: []uint16{
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA,
tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
tls.TLS_RSA_WITH_RC4_128_SHA,
tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
tls.TLS_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
},
PreferServerCipherSuites: true,
},
}
rp := &httputil.ReverseProxy{
Director: func(req *http.Request) {
log.Printf("Processing request to: %s", req.URL)
},
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
// Use a TLS session cache to minimize TLS connection establishment
// Requires Go 1.3+
ClientSessionCache: tls.NewLRUClientSessionCache(SESSIONS_TO_CACHE),
},
},
}
handler, err := Wrap(rp, cryptoConfig)
if err != nil {
log.Fatalf("Unable to wrap reverse proxy: %s", err)
}
server := &http.Server{
Addr: HTTP_ADDR,
Handler: handler,
ReadTimeout: 10 * time.Hour,
WriteTimeout: 10 * time.Hour,
}
go func() {
log.Printf("About to start HTTP proxy at %s", HTTP_ADDR)
if err := server.ListenAndServe(); err != nil {
log.Fatalf("Unable to start HTTP proxy: %s", err)
}
exampleWg.Done()
}()
return
}
func New() {
flag.Parse()
LoadConfig(*configPath)
server := &Server{
Docs: make(map[string]*Doc, 0),
sortedDocs: make([]*Doc, 0),
DocLock: &sync.RWMutex{},
Tags: make(map[string][]*Doc, 0),
TagLock: &sync.RWMutex{},
StaticFiles: make([]string, 0),
Version: Version,
StopWatch: make(chan bool)}
server.Reset()
if err := server.LoadTempalte(); err != nil {
log.Fatal(err)
}
server.LoadStaticFiles()
server.LoadAllDocs()
if err := server.SaveAllDocs(); err != nil {
log.Fatal(err)
}
server.MakeHome()
log.Print(server.MakeSitemap())
log.Print(Cfg)
server.Watch()
server.ConfigWatch()
admin := map[string]http.HandlerFunc{
".add": server.Add,
".edit": server.Edit,
".quit": server.Quit,
".upload": server.UploadMedia,
}
for k, v := range admin {
http.HandleFunc(path.Join(Cfg.BasePath, k), server.auth(v))
}
http.Handle("/", http.FileServer(http.Dir(Cfg.PublicPath)))
if Cfg.TLSKeyFile != "" && Cfg.TLSCertFile != "" {
tls_server := http.Server{Addr: Cfg.Addr, Handler: nil,
TLSConfig: &tls.Config{ClientSessionCache: tls.NewLRUClientSessionCache(100)}}
log.Fatal(tls_server.ListenAndServeTLS(Cfg.TLSCertFile, Cfg.TLSKeyFile))
} else {
log.Fatal(http.ListenAndServe(Cfg.Addr, nil))
}
}
func New(domain string, email string, apiToken string) *Util {
client := dnsimple.NewClient(apiToken, email)
// Set a longish timeout on the HTTP client just in case
client.HttpClient = &http.Client{
Timeout: 5 * time.Minute,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
ClientSessionCache: tls.NewLRUClientSessionCache(1000),
},
},
}
return &Util{client, domain}
}
func NewTLSServer(logger *util.Logger, caPool *CAPool, kvStore *kvstore.KVStore, hostname string) *TLSServer {
inst := &TLSServer{
ServerNode: ch.NewServerNode(hostname),
Logger: logger,
ControlChannel: make(chan (int)),
StatusChannel: make(chan (int)),
Connections: map[[16]byte]*Peer{},
SessionCache: tls.NewLRUClientSessionCache(1024),
KVStore: kvStore,
CAPool: caPool,
}
return inst
}
func New(domain string, username string, apiKey string) *Util {
client := cloudflare.NewClient(username, apiKey)
// Set a longish timeout on the HTTP client just in case
client.Http = &http.Client{
Timeout: 5 * time.Minute,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
ClientSessionCache: tls.NewLRUClientSessionCache(1000),
},
},
}
return &Util{client, domain}
}
func New(id string, key string, httpClient *http.Client) *cloudfront.CloudFront {
creds := aws.Creds(id, key, "")
// Set a longish timeout on the HTTP client just in case
if httpClient == nil {
httpClient = &http.Client{
Timeout: 5 * time.Minute,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
ClientSessionCache: tls.NewLRUClientSessionCache(1000),
},
},
}
}
return cloudfront.New(creds, "", httpClient)
}
func post(params map[string]interface{}) error {
buffer := bytes.NewBufferString("")
if err := tmpl.Execute(buffer, params); err != nil {
log.Printf("Airbrake error: %s", err)
return err
}
if Verbose {
log.Printf("Airbrake payload for endpoint %s: %s", Endpoint, buffer)
}
var httpClient = &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: &tls.Config{
ClientSessionCache: tls.NewLRUClientSessionCache(1024),
},
MaxIdleConnsPerHost: 100,
ResponseHeaderTimeout: 10 * time.Second,
},
Timeout: 10 * time.Second,
}
response, err := httpClient.Post(Endpoint, "text/xml", buffer)
if err != nil {
log.Printf("Airbrake error: %s", err)
return err
}
if Verbose {
body, _ := ioutil.ReadAll(response.Body)
log.Printf("response: %s", body)
}
response.Body.Close()
if Verbose {
log.Printf("Airbrake post: %s status code: %d", params["Error"], response.StatusCode)
}
return nil
}
// tlsConfig builds a tls.Config for dialing the upstream host. Constructed
// tls.Configs are cached on a per-masquerade basis to enable client session
// caching and reduce the amount of PEM certificate parsing.
func (d *direct) tlsConfig(m *Masquerade) *tls.Config {
d.tlsConfigsMutex.Lock()
defer d.tlsConfigsMutex.Unlock()
tlsConfig := d.tlsConfigs[m.Domain]
if tlsConfig == nil {
tlsConfig = &tls.Config{
ClientSessionCache: tls.NewLRUClientSessionCache(1000),
InsecureSkipVerify: false,
ServerName: m.Domain,
RootCAs: d.certPool,
}
d.tlsConfigs[m.Domain] = tlsConfig
}
return tlsConfig
}
// resetProxiedClient reconfigures the host so attempts to proxy through it,
// for the sake of checking whether it's up and serving, will use the given
// port. Valid port values are "443", in which case we'll access the proxy
// through HTTPS, or "80", for an unencrypted connection.
//
// This is necessary because when peerscanner starts up and gets the list of
// hosts from the various DNS/CDN services, it has no way to know what port
// these servers are listening at. So we want to try both until one works, or
// until the server first registers with peerscanner, advertising which port
// it uses.
func (h *host) resetProxiedClient(port string) {
var dial func(addr string) (net.Conn, error)
if port == "80" {
dial = func(addr string) (net.Conn, error) {
dialer := net.Dialer{Timeout: dialTimeout}
return dialer.Dial("tcp", h.ip+":80")
}
} else if port == "443" {
dial = func(addr string) (net.Conn, error) {
return tlsdialer.DialWithDialer(&net.Dialer{
Timeout: dialTimeout,
}, "tcp", h.ip+":443", true, &tls.Config{
InsecureSkipVerify: true,
// Cache TLS sessions
ClientSessionCache: tls.NewLRUClientSessionCache(1000),
})
}
} else {
log.Errorf("Unsupported port: %v", port)
return
}
h.proxiedClient = &http.Client{
Transport: &http.Transport{
Dial: func(network, addr string) (net.Conn, error) {
return enproxy.Dial(addr, &enproxy.Config{
DialProxy: dial,
NewRequest: func(upstreamHost string, method string, body io.Reader) (req *http.Request, err error) {
return http.NewRequest(method, "http://"+h.ip+"/", body)
},
OnFirstResponse: func(resp *http.Response) {
h.reportedHostMutex.Lock()
h.reportedHost = resp.Header.Get(enproxy.X_ENPROXY_PROXY_HOST)
h.reportedHostMutex.Unlock()
},
})
},
DisableKeepAlives: true,
},
Timeout: requestTimeout,
}
}
func NewDB(URL, CertPool, ClientCert string, HPKP []string) (*HTTPDB, error) {
remote, err := url.Parse(URL)
if err != nil {
return nil, err
}
db := &HTTPDB{
sni: remote.Host,
url: URL,
CertPool: CertPool,
ClientCert: ClientCert,
client: &http.Client{},
tlsconfig: &tls.Config{
ServerName: remote.Host,
MinVersion: tls.VersionTLS12,
MaxVersion: tls.VersionTLS12,
ClientSessionCache: tls.NewLRUClientSessionCache(MaxConn),
CurvePreferences: []tls.CurveID{
tls.CurveP521,
tls.CurveP384,
tls.CurveP256,
},
CipherSuites: []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
},
},
hpkp: make(map[string]bool),
}
for _, k := range HPKP {
db.hpkp[k] = true
}
db.client.Transport = &http.Transport{
MaxIdleConnsPerHost: MaxConn,
DialTLS: db.DialerTLS,
}
return db, nil
}
func NewFilter(config *Config) (filters.Filter, error) {
d := &dialer.Dialer{
Dialer: net.Dialer{
KeepAlive: time.Duration(config.Transport.Dialer.KeepAlive) * time.Second,
Timeout: time.Duration(config.Transport.Dialer.Timeout) * time.Second,
DualStack: config.Transport.Dialer.DualStack,
},
RetryTimes: config.Transport.Dialer.RetryTimes,
RetryDelay: time.Duration(config.Transport.Dialer.RetryDelay*1000) * time.Second,
DNSCache: lrucache.NewLRUCache(config.Transport.Dialer.DNSCacheSize),
DNSCacheExpiry: time.Duration(config.Transport.Dialer.DNSCacheExpiry) * time.Second,
LoopbackAddrs: make(map[string]struct{}),
}
d.LoopbackAddrs = make(map[string]struct{})
d.LoopbackAddrs["127.0.0.1"] = struct{}{}
d.LoopbackAddrs["::1"] = struct{}{}
if addrs, err := net.InterfaceAddrs(); err == nil {
for _, addr := range addrs {
switch addr.Network() {
case "ip":
d.LoopbackAddrs[addr.String()] = struct{}{}
}
}
}
tr := &http.Transport{
Dial: d.Dial,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: config.Transport.TLSClientConfig.InsecureSkipVerify,
ClientSessionCache: tls.NewLRUClientSessionCache(config.Transport.TLSClientConfig.ClientSessionCacheSize),
},
TLSHandshakeTimeout: time.Duration(config.Transport.TLSHandshakeTimeout) * time.Second,
MaxIdleConnsPerHost: config.Transport.MaxIdleConnsPerHost,
DisableCompression: config.Transport.DisableCompression,
}
return &Filter{
Config: *config,
transport: tr,
}, nil
}
func initTransport(artDetails *config.ArtifactoryDetails) error {
// Remove once SystemCertPool supports windows
caCertPool, err := LoadSystemRoots()
err = cliutils.CheckError(err)
if err != nil {
return err
}
err = loadCertificates(caCertPool)
if err != nil {
return err
}
// Setup HTTPS client
tlsConfig := &tls.Config{
RootCAs: caCertPool,
ClientSessionCache: tls.NewLRUClientSessionCache(1)}
tlsConfig.BuildNameToCertificate()
artDetails.Transport = &http.Transport{TLSClientConfig: tlsConfig}
return nil
}
// NewTLSSessionCache validates parameters and creates a new TLS session cache
func NewTLSSessionCache(s *TLSSessionCache) (tls.ClientSessionCache, error) {
cacheType := s.Type
if cacheType == "" {
cacheType = LRUCacheType
}
if cacheType != LRUCacheType {
return nil, fmt.Errorf("unsupported session cache type: %v", s.Type)
}
var capacity int
if params := s.Settings; params != nil {
if params.Capacity < 0 {
return nil, fmt.Errorf("bad LRU capacity: %v", params.Capacity)
} else if params.Capacity == 0 {
capacity = DefaultLRUCapacity
} else {
capacity = params.Capacity
}
}
return tls.NewLRUClientSessionCache(capacity), nil
}
// tlsConfig builds a tls.Config for dialing the upstream host. Constructed
// tls.Configs are cached on a per-masquerade basis to enable client session
// caching and reduce the amount of PEM certificate parsing.
func (d *dialer) tlsConfig(masquerade *Masquerade) *tls.Config {
d.tlsConfigsMutex.Lock()
defer d.tlsConfigsMutex.Unlock()
serverName := d.Host
if masquerade != nil {
serverName = masquerade.Domain
}
tlsConfig := d.tlsConfigs[serverName]
if tlsConfig == nil {
tlsConfig = &tls.Config{
ClientSessionCache: tls.NewLRUClientSessionCache(1000),
InsecureSkipVerify: d.InsecureSkipVerify,
ServerName: serverName,
RootCAs: d.RootCAs,
}
d.tlsConfigs[serverName] = tlsConfig
}
return tlsConfig
}
func initializeEtcdStoreClient(logger lager.Logger, etcdOptions *etcddb.ETCDOptions) etcddb.StoreClient {
var etcdClient *etcdclient.Client
var tr *http.Transport
if etcdOptions.IsSSL {
if etcdOptions.CertFile == "" || etcdOptions.KeyFile == "" {
logger.Fatal("failed-to-construct-etcd-tls-client", errors.New("Require both cert and key path"))
}
var err error
etcdClient, err = etcdclient.NewTLSClient(etcdOptions.ClusterUrls, etcdOptions.CertFile, etcdOptions.KeyFile, etcdOptions.CAFile)
if err != nil {
logger.Fatal("failed-to-construct-etcd-tls-client", err)
}
tlsCert, err := tls.LoadX509KeyPair(etcdOptions.CertFile, etcdOptions.KeyFile)
if err != nil {
logger.Fatal("failed-to-construct-etcd-tls-client", err)
}
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{tlsCert},
InsecureSkipVerify: true,
ClientSessionCache: tls.NewLRUClientSessionCache(etcdOptions.ClientSessionCacheSize),
}
tr = &http.Transport{
TLSClientConfig: tlsConfig,
Dial: etcdClient.DefaultDial,
MaxIdleConnsPerHost: etcdOptions.MaxIdleConnsPerHost,
}
etcdClient.SetTransport(tr)
etcdClient.AddRootCA(etcdOptions.CAFile)
} else {
etcdClient = etcdclient.NewClient(etcdOptions.ClusterUrls)
}
etcdClient.SetConsistency(etcdclient.STRONG_CONSISTENCY)
return etcddb.NewStoreClient(etcdClient)
}
func initializeEtcdClient(etcdOptions *ETCDOptions) *etcd.Client {
var etcdClient *etcd.Client
var tr *http.Transport
if etcdOptions.IsSSL {
if etcdOptions.CertFile == "" || etcdOptions.KeyFile == "" {
panic(errors.New("Require both cert and key path"))
}
var err error
etcdClient, err = etcd.NewTLSClient(etcdOptions.ClusterUrls, etcdOptions.CertFile, etcdOptions.KeyFile, etcdOptions.CAFile)
if err != nil {
panic(err)
}
tlsCert, err := tls.LoadX509KeyPair(etcdOptions.CertFile, etcdOptions.KeyFile)
if err != nil {
panic(err)
}
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{tlsCert},
InsecureSkipVerify: true,
ClientSessionCache: tls.NewLRUClientSessionCache(etcdOptions.ClientSessionCacheSize),
}
tr = &http.Transport{
TLSClientConfig: tlsConfig,
Dial: etcdClient.DefaultDial,
MaxIdleConnsPerHost: etcdOptions.MaxIdleConnsPerHost,
}
etcdClient.SetTransport(tr)
} else {
etcdClient = etcd.NewClient(etcdOptions.ClusterUrls)
}
etcdClient.SetConsistency(etcd.STRONG_CONSISTENCY)
return etcdClient
}
func NewFilter(config *Config) (filters.Filter, error) {
d := &Dailer{}
d.Timeout = time.Duration(config.Dialer.Timeout) * time.Second
d.KeepAlive = time.Duration(config.Dialer.KeepAlive) * time.Second
d.DNSCache = lrucache.NewMultiLRUCache(4, uint(config.DNSCache.Size))
d.DNSCacheExpires = time.Duration(config.DNSCache.Expires) * time.Second
d.Blacklist = make(map[string]struct{})
// d.Blacklist["127.0.0.1"] = struct{}{}
d.Blacklist["::1"] = struct{}{}
if addrs, err := net.InterfaceAddrs(); err == nil {
for _, addr := range addrs {
switch addr.Network() {
case "ip":
d.Blacklist[addr.String()] = struct{}{}
}
}
}
// glog.V(2).Infof("add blacklist=%v to direct filter", d.Blacklist)
return &Filter{
transport: &http.Transport{
Dial: d.Dial,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: false,
ClientSessionCache: tls.NewLRUClientSessionCache(1000),
},
TLSHandshakeTimeout: time.Duration(config.Transport.TLSHandshakeTimeout) * time.Second,
MaxIdleConnsPerHost: config.Transport.MaxIdleConnsPerHost,
DisableCompression: config.Transport.DisableCompression,
},
ratelimt: RateLimit{
Threshold: int64(config.RateLimit.Threshold),
Capacity: int64(config.RateLimit.Capacity),
Rate: float64(config.RateLimit.Rate),
},
}, nil
}
func NewFilter(config *Config) (filters.Filter, error) {
servers := make([]Server, 0)
for _, s := range config.Servers {
u, err := url.Parse(s.URL)
if err != nil {
return nil, err
}
server := Server{
URL: u,
Password: s.Password,
SSLVerify: s.SSLVerify,
Host: s.Host,
}
servers = append(servers, server)
}
d := &dialer.Dialer{
Dialer: net.Dialer{
KeepAlive: time.Duration(config.Transport.Dialer.KeepAlive) * time.Second,
Timeout: time.Duration(config.Transport.Dialer.Timeout) * time.Second,
DualStack: config.Transport.Dialer.DualStack,
},
RetryTimes: config.Transport.Dialer.RetryTimes,
RetryDelay: time.Duration(config.Transport.Dialer.RetryDelay*1000) * time.Second,
DNSCache: lrucache.NewLRUCache(config.Transport.Dialer.DNSCacheSize),
DNSCacheExpiry: time.Duration(config.Transport.Dialer.DNSCacheExpiry) * time.Second,
LoopbackAddrs: nil,
Level: 2,
}
for _, server := range servers {
if server.Host != "" {
host := server.URL.Host
if _, _, err := net.SplitHostPort(host); err != nil {
if server.URL.Scheme == "https" {
host += ":443"
} else {
host += ":80"
}
}
host1 := server.Host
if _, _, err := net.SplitHostPort(host1); err != nil {
if server.URL.Scheme == "https" {
host1 += ":443"
} else {
host1 += ":80"
}
}
d.DNSCache.Set(host, host1, time.Time{})
}
}
tr := &http.Transport{
Dial: d.Dial,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: false,
ClientSessionCache: tls.NewLRUClientSessionCache(1000),
},
TLSHandshakeTimeout: time.Duration(config.Transport.TLSHandshakeTimeout) * time.Second,
MaxIdleConnsPerHost: config.Transport.MaxIdleConnsPerHost,
}
if tr.TLSClientConfig != nil {
err := http2.ConfigureTransport(tr)
if err != nil {
glog.Warningf("PHP: Error enabling Transport HTTP/2 support: %v", err)
}
}
return &Filter{
Config: *config,
Transport: &Transport{
RoundTripper: tr,
Servers: servers,
},
Sites: helpers.NewHostMatcher(config.Sites),
}, nil
}
func NewFilter(config *Config) (filters.Filter, error) {
dnsServers := make([]net.IP, 0)
for _, s := range config.DNSServers {
if ip := net.ParseIP(s); ip != nil {
dnsServers = append(dnsServers, ip)
}
}
GoogleG2PKP, err := base64.StdEncoding.DecodeString(config.GoogleG2PKP)
if err != nil {
return nil, err
}
googleTLSConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true,
ServerName: "www.microsoft.com",
ClientSessionCache: tls.NewLRUClientSessionCache(config.TLSConfig.ClientSessionCacheSize),
CipherSuites: []uint16{
tls.TLS_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
},
}
switch config.TLSConfig.Version {
case "TLSv12", "TLSv1.2":
googleTLSConfig.MinVersion = tls.VersionTLS12
case "TLSv11", "TLSv1.1":
googleTLSConfig.MinVersion = tls.VersionTLS11
default:
googleTLSConfig.MinVersion = tls.VersionTLS10
}
pickupCiphers := func(names []string) []uint16 {
ciphers := make([]uint16, 0)
for _, name := range names {
cipher := helpers.Cipher(name)
if cipher == 0 {
glog.Fatalf("GAE: cipher %#v is not supported.", name)
}
ciphers = append(ciphers, cipher)
}
helpers.ShuffleUint16s(ciphers)
ciphers = ciphers[:1+rand.Intn(len(ciphers))]
ciphers1 := []uint16{}
for _, name := range []string{
"TLS_RSA_WITH_AES_256_CBC_SHA256",
"TLS_RSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
} {
if !helpers.ContainsString(names, name) {
if c := helpers.Cipher(name); c != 0 {
ciphers1 = append(ciphers1, c)
}
}
}
helpers.ShuffleUint16s(ciphers1)
ciphers1 = ciphers1[:rand.Intn(len(ciphers1))]
ciphers = append(ciphers, ciphers1...)
helpers.ShuffleUint16s(ciphers)
return ciphers
}
googleTLSConfig.CipherSuites = pickupCiphers(config.TLSConfig.Ciphers)
if len(config.TLSConfig.ServerName) > 0 {
googleTLSConfig.ServerName = config.TLSConfig.ServerName[rand.Intn(len(config.TLSConfig.ServerName))]
}
if !config.DisableHTTP2 {
googleTLSConfig.NextProtos = []string{"h2", "h2-14", "http/1.1"}
}
if config.Site2Alias == nil {
config.Site2Alias = make(map[string]string)
}
for key, value := range config.SiteToAlias {
config.Site2Alias[key] = value
}
config.SiteToAlias = config.Site2Alias
hostmap := map[string][]string{}
for key, value := range config.HostMap {
hostmap[key] = helpers.UniqueStrings(value)
}
d := &net.Dialer{
KeepAlive: time.Duration(config.Transport.Dialer.KeepAlive) * time.Second,
Timeout: time.Duration(config.Transport.Dialer.Timeout) * time.Second,
DualStack: config.Transport.Dialer.DualStack,
}
md := &dialer.MultiDialer{
Dialer: d,
DisableIPv6: config.DisableIPv6,
ForceIPv6: config.ForceIPv6,
SSLVerify: config.SSLVerify,
EnableRemoteDNS: config.EnableRemoteDNS,
LogToStderr: flag.Lookup("logtostderr") != nil,
TLSConfig: nil,
SiteToAlias: helpers.NewHostMatcherWithString(config.SiteToAlias),
IPBlackList: lrucache.NewLRUCache(1024),
HostMap: hostmap,
GoogleTLSConfig: googleTLSConfig,
GoogleG2PKP: GoogleG2PKP,
DNSServers: dnsServers,
DNSCache: lrucache.NewLRUCache(config.Transport.Dialer.DNSCacheSize),
DNSCacheExpiry: time.Duration(config.Transport.Dialer.DNSCacheExpiry) * time.Second,
TLSConnDuration: lrucache.NewLRUCache(8192),
TLSConnError: lrucache.NewLRUCache(8192),
TLSConnReadBuffer: config.Transport.Dialer.SocketReadBuffer,
ConnExpiry: 5 * time.Minute,
Level: config.Transport.Dialer.Level,
}
for _, ip := range config.IPBlackList {
md.IPBlackList.Set(ip, struct{}{}, time.Time{})
}
var tr http.RoundTripper
t1 := &http.Transport{
Dial: md.Dial,
DialTLS: md.DialTLS,
DisableKeepAlives: config.Transport.DisableKeepAlives,
DisableCompression: config.Transport.DisableCompression,
ResponseHeaderTimeout: time.Duration(config.Transport.ResponseHeaderTimeout) * time.Second,
IdleConnTimeout: time.Duration(config.Transport.IdleConnTimeout) * time.Second,
MaxIdleConnsPerHost: config.Transport.MaxIdleConnsPerHost,
}
if config.Transport.Proxy.Enabled {
fixedURL, err := url.Parse(config.Transport.Proxy.URL)
if err != nil {
glog.Fatalf("url.Parse(%#v) error: %s", config.Transport.Proxy.URL, err)
}
dialer, err := proxy.FromURL(fixedURL, d, &dialer.MultiResolver{md})
if err != nil {
glog.Fatalf("proxy.FromURL(%#v) error: %s", fixedURL.String(), err)
}
t1.Dial = dialer.Dial
t1.DialTLS = nil
t1.Proxy = nil
t1.TLSClientConfig = md.GoogleTLSConfig
}
switch {
case config.DisableHTTP2 && config.ForceHTTP2:
glog.Fatalf("GAE: DisableHTTP2=%v and ForceHTTP2=%v is conflict!", config.DisableHTTP2, config.ForceHTTP2)
case config.Transport.Proxy.Enabled && config.ForceHTTP2:
glog.Fatalf("GAE: Proxy.Enabled=%v and ForceHTTP2=%v is conflict!", config.Transport.Proxy.Enabled, config.ForceHTTP2)
case config.ForceHTTP2:
tr = &http2.Transport{
DialTLS: md.DialTLS2,
TLSClientConfig: md.GoogleTLSConfig,
DisableCompression: config.Transport.DisableCompression,
}
case !config.DisableHTTP2:
err := http2.ConfigureTransport(t1)
if err != nil {
glog.Warningf("GAE: Error enabling Transport HTTP/2 support: %v", err)
}
tr = t1
default:
tr = t1
}
forceHTTPSMatcherStrings := make([]string, 0)
for key, value := range config.SiteToAlias {
if strings.HasPrefix(value, "google_") {
forceHTTPSMatcherStrings = append(forceHTTPSMatcherStrings, key)
}
}
forceGAEStrings := make([]string, 0)
forceGAESuffixs := make([]string, 0)
forceGAEMatcherStrings := make([]string, 0)
for _, s := range config.ForceGAE {
if strings.Contains(s, "/") {
if strings.HasSuffix(s, "$") {
forceGAESuffixs = append(forceGAESuffixs, strings.TrimRight(s, "$"))
} else {
forceGAEStrings = append(forceGAEStrings, s)
}
} else {
forceGAEMatcherStrings = append(forceGAEMatcherStrings, s)
}
}
if config.EnableDeadProbe && !config.Transport.Proxy.Enabled {
go func() {
probe := func() {
req, _ := http.NewRequest(http.MethodGet, "https://clients3.google.com/generate_204", nil)
ctx, cancel := context.WithTimeout(req.Context(), 3*time.Second)
defer cancel()
req = req.WithContext(ctx)
resp, err := tr.RoundTrip(req)
if resp != nil && resp.Body != nil {
glog.V(3).Infof("GAE EnableDeadProbe \"%s %s\" %d -", req.Method, req.URL.String(), resp.StatusCode)
resp.Body.Close()
}
if err != nil {
glog.V(2).Infof("GAE EnableDeadProbe \"%s %s\" error: %v", req.Method, req.URL.String(), err)
s := strings.ToLower(err.Error())
if strings.HasPrefix(s, "net/http: request canceled") || strings.Contains(s, "timeout") {
helpers.TryCloseConnections(tr)
}
}
}
for {
time.Sleep(time.Duration(5+rand.Intn(6)) * time.Second)
probe()
}
}()
}
helpers.ShuffleStrings(config.AppIDs)
f := &Filter{
Config: *config,
GAETransport: &Transport{
RoundTripper: tr,
MultiDialer: md,
Servers: NewServers(config.AppIDs,
config.Password,
config.SSLVerify,
time.Duration(config.Transport.ResponseHeaderTimeout-4)*time.Second),
RetryDelay: time.Duration(config.Transport.RetryDelay*1000) * time.Second,
RetryTimes: config.Transport.RetryTimes,
},
DirectTransport: tr,
ForceHTTPSMatcher: helpers.NewHostMatcher(forceHTTPSMatcherStrings),
ForceGAEMatcher: helpers.NewHostMatcher(forceGAEMatcherStrings),
ForceGAEStrings: forceGAEStrings,
ForceGAESuffixs: forceGAESuffixs,
ForceDeflateMatcher: helpers.NewHostMatcher(config.ForceDeflate),
FakeOptionsMatcher: helpers.NewHostMatcherWithStrings(config.FakeOptions),
SiteMatcher: helpers.NewHostMatcher(config.Sites),
DirectSiteMatcher: helpers.NewHostMatcherWithString(config.Site2Alias),
}
if config.Transport.Proxy.Enabled {
f.GAETransport.MultiDialer = nil
}
return f, nil
}
func NewFilter(config *Config) (filters.Filter, error) {
d := &Dialer{}
d.Timeout = time.Duration(config.Dialer.Timeout) * time.Second
d.KeepAlive = time.Duration(config.Dialer.KeepAlive) * time.Second
d.Blacklist = make(map[string]struct{})
d.Window = config.Dialer.Window
d.Blacklist["127.0.0.1"] = struct{}{}
d.Blacklist["::1"] = struct{}{}
if addrs, err := net.InterfaceAddrs(); err == nil {
for _, addr := range addrs {
switch addr.Network() {
case "ip":
d.Blacklist[addr.String()] = struct{}{}
}
}
}
var err error
d.hosts = httpproxy.NewHostMatcherWithString(config.Hosts)
d.iplist, err = NewIplist(config.Iplist, config.DNS.Servers, config.DNS.Blacklist, d.DualStack)
if err != nil {
return nil, err
}
d.TLSConfig = &tls.Config{
InsecureSkipVerify: true,
ClientSessionCache: tls.NewLRUClientSessionCache(1000),
}
d.connTCPDuration = lrucache.NewMultiLRUCache(4, 4096)
d.connTLSDuration = lrucache.NewMultiLRUCache(4, 4096)
d.connExpireDuration = 5 * time.Minute
for _, name := range config.DNS.Expand {
if _, ok := config.Iplist[name]; ok {
go func(name string) {
t := time.Tick(3 * time.Minute)
for {
select {
case <-t:
d.iplist.ExpandList(name)
}
}
}(name)
}
}
return &Filter{
transport: &http.Transport{
Dial: d.Dial,
DialTLS: d.DialTLS,
DisableKeepAlives: config.Transport.DisableKeepAlives,
DisableCompression: config.Transport.DisableCompression,
TLSHandshakeTimeout: time.Duration(config.Transport.TLSHandshakeTimeout) * time.Second,
MaxIdleConnsPerHost: config.Transport.MaxIdleConnsPerHost,
},
dialer: d,
}, nil
}
func main() {
// Parse the flags
flag.Parse()
// Normalize the etcd path
if (*etcdPath)[0] != '/' {
*etcdPath = "/" + *etcdPath
}
if (*etcdPath)[len(*etcdPath)-1] == '/' {
*etcdPath = (*etcdPath)[:len(*etcdPath)-2]
}
// Transform etcdAddress into addresses
addresses := strings.Split(*etcdAddress, ",")
etcc = etcd.NewClient(addresses)
// Set up a new logger
log = logrus.New()
// Set the formatter depending on the passed flag's value
if *logFormatterType == "text" {
log.Formatter = &logrus.TextFormatter{
ForceColors: *logForceColors,
}
} else if *logFormatterType == "json" {
log.Formatter = &logrus.JSONFormatter{}
}
// Create a new TLS config
tlsConfig = &tls.Config{
Certificates: certs,
MinVersion: tls.VersionTLS10,
MaxVersion: tls.VersionTLS12,
CipherSuites: []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_RSA_WITH_AES_128_CBC_SHA,
},
PreferServerCipherSuites: true,
ClientSessionCache: tls.NewLRUClientSessionCache(*sessionCacheSize),
}
tlsConfig.BuildNameToCertificate()
// Load the settings
loadSettings()
// Start the HTTPS server
go func() {
conn, err := net.Listen("tcp", *tlsBind)
if err != nil {
log.WithFields(logrus.Fields{
"error": err.Error(),
"address": *tlsBind,
}).Fatal("Unable to prepare a listener for the HTTPS server")
}
listener := tls.NewListener(conn, tlsConfig)
server := &http.Server{
Handler: http.HandlerFunc(handler),
}
err = server.Serve(listener)
if err != nil {
log.WithFields(logrus.Fields{
"error": err.Error(),
}).Fatal("Error while serving a HTTPS listener")
}
}()
// Start the HTTP server
go func() {
err := http.ListenAndServe(*rawBind, http.HandlerFunc(handler))
if err != nil {
log.WithFields(logrus.Fields{
"error": err.Error(),
"address": *rawBind,
}).Fatal("Unable to bind the HTTP server")
}
}()
// Watch for changes in etcd
receiver := make(chan *etcd.Response)
stop := make(chan bool)
go func() {
for {
// Wait for a change
<-receiver
// Log debug information
log.Info("Reloading settings")
// Reload the settings
loadSettings()
}
}()
_, err := etcc.Watch(*etcdPath, 0, true, receiver, stop)
if err != nil {
log.WithFields(logrus.Fields{
"error": err.Error(),
}).Fatal("etcd watcher errored")
}
}
host, projectId, key,
)
}
type filter func(*Notice) *Notice
var httpClient = &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: 15 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: &tls.Config{
ClientSessionCache: tls.NewLRUClientSessionCache(1024),
},
MaxIdleConnsPerHost: 10,
ResponseHeaderTimeout: 10 * time.Second,
},
Timeout: 10 * time.Second,
}
type Notifier struct {
projectId int64
projectKey string
createNoticeURL string
Client *http.Client
context map[string]string
n = defaultReadBufferSize
}
return bufio.NewReaderSize(conn, n)
}
br := v.(*bufio.Reader)
br.Reset(conn)
return br
}
func (c *HostClient) releaseReader(br *bufio.Reader) {
c.readerPool.Put(br)
}
var defaultTLSConfig = &tls.Config{
InsecureSkipVerify: true,
ClientSessionCache: tls.NewLRUClientSessionCache(0),
}
func (c *HostClient) nextAddr() string {
c.addrsLock.Lock()
if c.addrs == nil {
c.addrs = strings.Split(c.Addr, ",")
}
addr := c.addrs[0]
if len(c.addrs) > 1 {
addr = c.addrs[c.addrIdx%uint32(len(c.addrs))]
c.addrIdx++
}
c.addrsLock.Unlock()
return addr
}