func (d *MultiDialer) LookupAlias(alias string) (addrs []string, err error) {
names, ok := d.HostMap[alias]
if !ok {
return nil, fmt.Errorf("alias %#v not exists", alias)
}
seen := make(map[string]struct{}, 0)
expiry := time.Now().Add(d.DNSCacheExpiry)
for _, name := range names {
var addrs0 []string
if net.ParseIP(name) != nil {
addrs0 = []string{name}
expiry = time.Time{}
} else if addrs1, ok := d.DNSCache.Get(name); ok {
addrs0 = addrs1.([]string)
} else {
if d.ForceIPv6 {
addrs0, err = d.LookupHost2(name, d.DNSServers[0])
if err != nil {
glog.Warningf("LookupHost2(%#v, %#v) error: %s", name, d.DNSServers[0], err)
addrs0 = []string{}
}
} else {
addrs0, err = d.LookupHost(name)
if err != nil {
glog.Warningf("LookupHost(%#v) error: %s", name, err)
addrs0 = []string{}
}
}
glog.V(2).Infof("LookupHost(%#v) return %v", name, addrs0)
d.DNSCache.Set(name, addrs0, expiry)
}
for _, addr := range addrs0 {
seen[addr] = struct{}{}
}
}
if len(seen) == 0 {
return nil, err
}
addrs = make([]string, 0)
for addr, _ := range seen {
if _, ok := d.IPBlackList.GetQuiet(addr); ok {
continue
}
addrs = append(addrs, addr)
}
if len(addrs) == 0 {
glog.Errorf("MULTIDIALER: LookupAlias(%#v) have no good ip addrs", alias)
return nil, fmt.Errorf("MULTIDIALER: LookupAlias(%#v) have no good ip addrs", alias)
}
return addrs, nil
}
func (f *Filter) Request(ctx context.Context, req *http.Request) (context.Context, *http.Request, error) {
if req.Method != http.MethodGet || strings.Contains(req.URL.RawQuery, "range=") {
return ctx, req, nil
}
if r := req.Header.Get("Range"); r == "" {
switch {
case f.SiteMatcher.Match(req.Host):
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", 0, f.MaxSize))
glog.V(2).Infof("AUTORANGE Sites rule matched, add %s for\"%s\"", req.Header.Get("Range"), req.URL.String())
ctx = filters.WithBool(ctx, "autorange.site", true)
default:
glog.V(3).Infof("AUTORANGE ignore preserved empty range for %#v", req.URL)
}
} else {
ctx = filters.WithBool(ctx, "autorange.default", true)
parts := strings.Split(r, "=")
switch parts[0] {
case "bytes":
parts1 := strings.Split(parts[1], "-")
if start, err := strconv.Atoi(parts1[0]); err == nil {
if end, err := strconv.Atoi(parts1[1]); err != nil || end-start > f.MaxSize {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, start+f.MaxSize))
glog.V(2).Infof("AUTORANGE Default rule matched, change %s to %s for\"%s\"", r, req.Header.Get("Range"), req.URL.String())
}
}
default:
glog.Warningf("AUTORANGE Default rule matched, but cannot support %#v range for \"%s\"", r, req.URL.String())
}
}
return ctx, req, nil
}
func forward(lconn net.Conn, raddr string) {
if err := exec.Command("wakeonlan", "18:66:DA:17:A2:95").Run(); err != nil {
glog.Warningf("exec wakeonlan error: %+v", err)
}
glog.Infof("try connect to %+v for %+v forwarding", raddr, lconn.RemoteAddr())
rconn, err := net.Dial("tcp", raddr)
if err != nil {
glog.Errorf("net.Dial(%#v) error: %v", raddr, err)
return
}
glog.Infof("forward %+v to %+v", lconn.RemoteAddr(), rconn.RemoteAddr())
go func() {
buf := bufpool.Get().([]byte)
defer bufpool.Put(buf)
defer rconn.Close()
defer lconn.Close()
io.CopyBuffer(rconn, lconn, buf)
}()
go func() {
buf := bufpool.Get().([]byte)
defer bufpool.Put(buf)
defer lconn.Close()
defer rconn.Close()
io.CopyBuffer(lconn, rconn, buf)
}()
}
func (l *listener) Accept() (c net.Conn, err error) {
l.once.Do(func() {
go func() {
var tempDelay time.Duration
for {
conn, err := l.ln.Accept()
l.lane <- connRacer{conn, err}
if err != nil {
if ne, ok := err.(net.Error); ok && ne.Temporary() {
if tempDelay == 0 {
tempDelay = 5 * time.Millisecond
} else {
tempDelay *= 2
}
if max := 1 * time.Second; tempDelay > max {
tempDelay = max
}
glog.Warningf("httpproxy.Listener: Accept error: %v; retrying in %v", err, tempDelay)
time.Sleep(tempDelay)
continue
}
return
}
}
}()
})
r := <-l.lane
if r.err != nil {
return r.conn, r.err
}
if l.keepAlivePeriod > 0 || l.readBufferSize > 0 || l.writeBufferSize > 0 {
if tc, ok := r.conn.(*net.TCPConn); ok {
if l.keepAlivePeriod > 0 {
tc.SetKeepAlive(true)
tc.SetKeepAlivePeriod(l.keepAlivePeriod)
}
if l.readBufferSize > 0 {
tc.SetReadBuffer(l.readBufferSize)
}
if l.writeBufferSize > 0 {
tc.SetWriteBuffer(l.writeBufferSize)
}
}
}
return r.conn, nil
}
func (d *MultiDialer) LookupAlias(alias string) (addrs []string, err error) {
names, ok := d.HostMap[alias]
if !ok {
return nil, fmt.Errorf("alias %#v not exists", alias)
}
seen := make(map[string]struct{}, 0)
for _, name := range names {
var addrs0 []string
if net.ParseIP(name) != nil {
addrs0 = []string{name}
} else {
addrs0, err = d.Resolver.LookupHost(name)
if err != nil {
glog.Warningf("LookupHost(%#v) error: %s", name, err)
addrs0 = []string{}
}
}
for _, addr := range addrs0 {
seen[addr] = struct{}{}
}
}
if len(seen) == 0 {
return nil, err
}
addrs = make([]string, 0)
for addr := range seen {
if _, ok := d.IPBlackList.GetQuiet(addr); ok {
continue
}
addrs = append(addrs, addr)
}
if len(addrs) == 0 {
glog.Errorf("MULTIDIALER: LookupAlias(%#v) have no good ip addrs", alias)
return nil, fmt.Errorf("MULTIDIALER: LookupAlias(%#v) have no good ip addrs", alias)
}
return addrs, nil
}
func (d *MultiDialer) Dial(network, address string) (net.Conn, error) {
glog.Warningf("MULTIDIALER Dial(%#v, %#v) with good_addrs=%d, bad_addrs=%d", network, address, d.TCPConnDuration.Len(), d.TCPConnError.Len())
switch network {
case "tcp", "tcp4", "tcp6":
if host, port, err := net.SplitHostPort(address); err == nil {
if alias0, ok := d.Site2Alias.Lookup(host); ok {
alias := alias0.(string)
if hosts, err := d.LookupAlias(alias); err == nil {
addrs := make([]string, len(hosts))
for i, host := range hosts {
addrs[i] = net.JoinHostPort(host, port)
}
if d.ForceIPv6 {
network = "tcp6"
}
return d.dialMulti(network, addrs)
}
}
}
default:
break
}
return d.Dialer.Dial(network, address)
}
func NewFilter(config *Config) (filters.Filter, error) {
dnsServers := make([]net.IP, 0)
for _, s := range config.DNSServers {
if ip := net.ParseIP(s); ip != nil {
dnsServers = append(dnsServers, ip)
}
}
GoogleG2PKP, err := base64.StdEncoding.DecodeString(config.GoogleG2PKP)
if err != nil {
return nil, err
}
googleTLSConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true,
ServerName: "www.microsoft.com",
ClientSessionCache: tls.NewLRUClientSessionCache(config.TLSConfig.ClientSessionCacheSize),
CipherSuites: []uint16{
tls.TLS_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
},
}
switch config.TLSConfig.Version {
case "TLSv12", "TLSv1.2":
googleTLSConfig.MinVersion = tls.VersionTLS12
case "TLSv11", "TLSv1.1":
googleTLSConfig.MinVersion = tls.VersionTLS11
default:
googleTLSConfig.MinVersion = tls.VersionTLS10
}
pickupCiphers := func(names []string) []uint16 {
ciphers := make([]uint16, 0)
for _, name := range names {
cipher := helpers.Cipher(name)
if cipher == 0 {
glog.Fatalf("GAE: cipher %#v is not supported.", name)
}
ciphers = append(ciphers, cipher)
}
helpers.ShuffleUint16s(ciphers)
ciphers = ciphers[:1+rand.Intn(len(ciphers))]
ciphers1 := []uint16{}
for _, name := range []string{
"TLS_RSA_WITH_AES_256_CBC_SHA256",
"TLS_RSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
} {
if !helpers.ContainsString(names, name) {
if c := helpers.Cipher(name); c != 0 {
ciphers1 = append(ciphers1, c)
}
}
}
helpers.ShuffleUint16s(ciphers1)
ciphers1 = ciphers1[:rand.Intn(len(ciphers1))]
ciphers = append(ciphers, ciphers1...)
helpers.ShuffleUint16s(ciphers)
return ciphers
}
googleTLSConfig.CipherSuites = pickupCiphers(config.TLSConfig.Ciphers)
if len(config.TLSConfig.ServerName) > 0 {
googleTLSConfig.ServerName = config.TLSConfig.ServerName[rand.Intn(len(config.TLSConfig.ServerName))]
}
if !config.DisableHTTP2 {
googleTLSConfig.NextProtos = []string{"h2", "h2-14", "http/1.1"}
}
if config.Site2Alias == nil {
config.Site2Alias = make(map[string]string)
}
for key, value := range config.SiteToAlias {
config.Site2Alias[key] = value
}
config.SiteToAlias = config.Site2Alias
hostmap := map[string][]string{}
for key, value := range config.HostMap {
hostmap[key] = helpers.UniqueStrings(value)
}
d := &net.Dialer{
KeepAlive: time.Duration(config.Transport.Dialer.KeepAlive) * time.Second,
Timeout: time.Duration(config.Transport.Dialer.Timeout) * time.Second,
DualStack: config.Transport.Dialer.DualStack,
}
md := &dialer.MultiDialer{
Dialer: d,
DisableIPv6: config.DisableIPv6,
ForceIPv6: config.ForceIPv6,
SSLVerify: config.SSLVerify,
EnableRemoteDNS: config.EnableRemoteDNS,
LogToStderr: flag.Lookup("logtostderr") != nil,
TLSConfig: nil,
SiteToAlias: helpers.NewHostMatcherWithString(config.SiteToAlias),
IPBlackList: lrucache.NewLRUCache(1024),
HostMap: hostmap,
GoogleTLSConfig: googleTLSConfig,
GoogleG2PKP: GoogleG2PKP,
DNSServers: dnsServers,
DNSCache: lrucache.NewLRUCache(config.Transport.Dialer.DNSCacheSize),
DNSCacheExpiry: time.Duration(config.Transport.Dialer.DNSCacheExpiry) * time.Second,
TLSConnDuration: lrucache.NewLRUCache(8192),
TLSConnError: lrucache.NewLRUCache(8192),
TLSConnReadBuffer: config.Transport.Dialer.SocketReadBuffer,
ConnExpiry: 5 * time.Minute,
Level: config.Transport.Dialer.Level,
}
for _, ip := range config.IPBlackList {
md.IPBlackList.Set(ip, struct{}{}, time.Time{})
}
var tr http.RoundTripper
t1 := &http.Transport{
Dial: md.Dial,
DialTLS: md.DialTLS,
DisableKeepAlives: config.Transport.DisableKeepAlives,
DisableCompression: config.Transport.DisableCompression,
ResponseHeaderTimeout: time.Duration(config.Transport.ResponseHeaderTimeout) * time.Second,
IdleConnTimeout: time.Duration(config.Transport.IdleConnTimeout) * time.Second,
MaxIdleConnsPerHost: config.Transport.MaxIdleConnsPerHost,
}
if config.Transport.Proxy.Enabled {
fixedURL, err := url.Parse(config.Transport.Proxy.URL)
if err != nil {
glog.Fatalf("url.Parse(%#v) error: %s", config.Transport.Proxy.URL, err)
}
dialer, err := proxy.FromURL(fixedURL, d, &dialer.MultiResolver{md})
if err != nil {
glog.Fatalf("proxy.FromURL(%#v) error: %s", fixedURL.String(), err)
}
t1.Dial = dialer.Dial
t1.DialTLS = nil
t1.Proxy = nil
t1.TLSClientConfig = md.GoogleTLSConfig
}
switch {
case config.DisableHTTP2 && config.ForceHTTP2:
glog.Fatalf("GAE: DisableHTTP2=%v and ForceHTTP2=%v is conflict!", config.DisableHTTP2, config.ForceHTTP2)
case config.Transport.Proxy.Enabled && config.ForceHTTP2:
glog.Fatalf("GAE: Proxy.Enabled=%v and ForceHTTP2=%v is conflict!", config.Transport.Proxy.Enabled, config.ForceHTTP2)
case config.ForceHTTP2:
tr = &http2.Transport{
DialTLS: md.DialTLS2,
TLSClientConfig: md.GoogleTLSConfig,
DisableCompression: config.Transport.DisableCompression,
}
case !config.DisableHTTP2:
err := http2.ConfigureTransport(t1)
if err != nil {
glog.Warningf("GAE: Error enabling Transport HTTP/2 support: %v", err)
}
tr = t1
default:
tr = t1
}
forceHTTPSMatcherStrings := make([]string, 0)
for key, value := range config.SiteToAlias {
if strings.HasPrefix(value, "google_") {
forceHTTPSMatcherStrings = append(forceHTTPSMatcherStrings, key)
}
}
forceGAEStrings := make([]string, 0)
forceGAESuffixs := make([]string, 0)
forceGAEMatcherStrings := make([]string, 0)
for _, s := range config.ForceGAE {
if strings.Contains(s, "/") {
if strings.HasSuffix(s, "$") {
forceGAESuffixs = append(forceGAESuffixs, strings.TrimRight(s, "$"))
} else {
forceGAEStrings = append(forceGAEStrings, s)
}
} else {
forceGAEMatcherStrings = append(forceGAEMatcherStrings, s)
}
}
if config.EnableDeadProbe && !config.Transport.Proxy.Enabled {
go func() {
probe := func() {
req, _ := http.NewRequest(http.MethodGet, "https://clients3.google.com/generate_204", nil)
ctx, cancel := context.WithTimeout(req.Context(), 3*time.Second)
defer cancel()
req = req.WithContext(ctx)
resp, err := tr.RoundTrip(req)
if resp != nil && resp.Body != nil {
glog.V(3).Infof("GAE EnableDeadProbe \"%s %s\" %d -", req.Method, req.URL.String(), resp.StatusCode)
resp.Body.Close()
}
if err != nil {
glog.V(2).Infof("GAE EnableDeadProbe \"%s %s\" error: %v", req.Method, req.URL.String(), err)
s := strings.ToLower(err.Error())
if strings.HasPrefix(s, "net/http: request canceled") || strings.Contains(s, "timeout") {
helpers.TryCloseConnections(tr)
}
}
}
for {
time.Sleep(time.Duration(5+rand.Intn(6)) * time.Second)
probe()
}
}()
}
helpers.ShuffleStrings(config.AppIDs)
f := &Filter{
Config: *config,
GAETransport: &Transport{
RoundTripper: tr,
MultiDialer: md,
Servers: NewServers(config.AppIDs,
config.Password,
config.SSLVerify,
time.Duration(config.Transport.ResponseHeaderTimeout-4)*time.Second),
RetryDelay: time.Duration(config.Transport.RetryDelay*1000) * time.Second,
RetryTimes: config.Transport.RetryTimes,
},
DirectTransport: tr,
ForceHTTPSMatcher: helpers.NewHostMatcher(forceHTTPSMatcherStrings),
ForceGAEMatcher: helpers.NewHostMatcher(forceGAEMatcherStrings),
ForceGAEStrings: forceGAEStrings,
ForceGAESuffixs: forceGAESuffixs,
ForceDeflateMatcher: helpers.NewHostMatcher(config.ForceDeflate),
FakeOptionsMatcher: helpers.NewHostMatcherWithStrings(config.FakeOptions),
SiteMatcher: helpers.NewHostMatcher(config.Sites),
DirectSiteMatcher: helpers.NewHostMatcherWithString(config.Site2Alias),
}
if config.Transport.Proxy.Enabled {
f.GAETransport.MultiDialer = nil
}
return f, nil
}
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
var tr http.RoundTripper = f.GAETransport
if req.URL.Scheme == "http" && f.ForceHTTPSMatcher.Match(req.Host) {
if !strings.HasPrefix(req.Header.Get("Referer"), "https://") {
u := strings.Replace(req.URL.String(), "http://", "https://", 1)
glog.V(2).Infof("GAE FORCEHTTPS get raw url=%v, redirect to %v", req.URL.String(), u)
resp := &http.Response{
StatusCode: http.StatusMovedPermanently,
Header: http.Header{
"Location": []string{u},
},
Request: req,
Close: true,
ContentLength: -1,
}
glog.V(2).Infof("%s \"GAE FORCEHTTPS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
if f.DirectSiteMatcher.Match(req.Host) {
switch req.URL.Path {
case "/url":
if rawurl := req.URL.Query().Get("url"); rawurl != "" {
if u, err := url.Parse(rawurl); err == nil {
if u.Scheme == "http" && f.ForceHTTPSMatcher.Match(u.Host) {
rawurl = strings.Replace(rawurl, "http://", "https://", 1)
}
}
glog.V(2).Infof("%s \"GAE REDIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, rawurl, req.Proto)
return ctx, &http.Response{
StatusCode: http.StatusFound,
Header: http.Header{
"Location": []string{rawurl},
},
Request: req,
Close: true,
ContentLength: -1,
}, nil
}
case "/books":
if req.URL.Host == "books.google.cn" {
rawurl := strings.Replace(req.URL.String(), "books.google.cn", "books.google.com", 1)
glog.V(2).Infof("%s \"GAE REDIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, rawurl, req.Proto)
return ctx, &http.Response{
StatusCode: http.StatusFound,
Header: http.Header{
"Location": []string{rawurl},
},
Request: req,
Close: true,
ContentLength: -1,
}, nil
}
}
if req.URL.Scheme != "http" && !f.shouldForceGAE(req) {
tr = f.DirectTransport
if s := req.Header.Get("Connection"); s != "" {
if s1 := strings.ToLower(s); s != s1 {
req.Header.Set("Connection", s1)
}
}
}
}
if tr != f.DirectTransport && req.Method == http.MethodOptions {
if v, ok := f.FakeOptionsMatcher.Lookup(req.Host); ok {
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{},
Request: req,
Close: false,
ContentLength: -1,
}
for _, s := range v.([]string) {
parts := strings.SplitN(s, ":", 2)
if len(parts) == 2 {
resp.Header.Add(parts[0], strings.TrimSpace(parts[1]))
}
}
if origin := req.Header.Get("Origin"); origin != "" {
resp.Header.Set("Access-Control-Allow-Origin", origin)
}
if headers := req.Header.Get("Access-Control-Request-Headers"); headers != "" {
resp.Header.Set("Access-Control-Allow-Headers", headers)
}
glog.V(2).Infof("%s \"GAE FAKEOPTIONS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
if tr == f.GAETransport {
switch strings.ToLower(req.Header.Get("Connection")) {
case "upgrade":
resp := &http.Response{
StatusCode: http.StatusForbidden,
Header: http.Header{
"X-WebSocket-Reject-Reason": []string{"Unsupported"},
},
Request: req,
Close: true,
ContentLength: 0,
}
glog.V(2).Infof("%s \"GAE FAKEWEBSOCKET %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
prefix := "FETCH"
if tr == f.DirectTransport {
prefix = "DIRECT"
}
resp, err := tr.RoundTrip(req)
if err != nil {
glog.Warningf("%s \"GAE %s %s %s %s\" error: %T(%v)", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, err, err)
if tr == f.DirectTransport {
if ne, ok := err.(interface {
Timeout() bool
}); ok && ne.Timeout() {
// f.MultiDialer.ClearCache()
helpers.CloseConnections(tr)
}
}
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
return ctx, nil, err
}
if tr == f.DirectTransport && resp.StatusCode >= http.StatusBadRequest {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
resp.Body.Close()
return ctx, nil, err
}
if addr, err := helpers.ReflectRemoteAddrFromResponse(resp); err == nil {
if ip, _, err := net.SplitHostPort(addr); err == nil {
var duration time.Duration
switch {
case resp.StatusCode == http.StatusBadGateway && bytes.Contains(body, []byte("Please try again in 30 seconds.")):
duration = 1 * time.Hour
case resp.StatusCode == http.StatusNotFound && bytes.Contains(body, []byte("<ins>That’s all we know.</ins>")):
server := resp.Header.Get("Server")
if server != "gws" && !strings.HasPrefix(server, "gvs") {
if md := f.GAETransport.MultiDialer; md != nil && md.TLSConnDuration.Len() > 10 {
duration = 5 * time.Minute
}
}
}
if duration > 0 && f.GAETransport.MultiDialer != nil {
glog.Warningf("GAE: %s StatusCode is %d, not a gws/gvs ip, add to blacklist for %v", ip, resp.StatusCode, duration)
f.GAETransport.MultiDialer.IPBlackList.Set(ip, struct{}{}, time.Now().Add(duration))
if !helpers.CloseConnectionByRemoteAddr(tr, addr) {
glog.Warningf("GAE: CloseConnectionByRemoteAddr(%T, %#v) failed.", tr, addr)
}
}
}
}
resp.Body.Close()
resp.Body = ioutil.NopCloser(bytes.NewReader(body))
}
if resp != nil && resp.Header != nil {
resp.Header.Del("Alt-Svc")
resp.Header.Del("Alternate-Protocol")
}
glog.V(2).Infof("%s \"GAE %s %s %s %s\" %d %s", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, err
}
func NewFilter(config *Config) (filters.Filter, error) {
servers := make([]Server, 0)
for _, s := range config.Servers {
u, err := url.Parse(s.URL)
if err != nil {
return nil, err
}
server := Server{
URL: u,
Password: s.Password,
SSLVerify: s.SSLVerify,
Host: s.Host,
}
servers = append(servers, server)
}
d0 := &net.Dialer{
KeepAlive: time.Duration(config.Transport.Dialer.KeepAlive) * time.Second,
Timeout: time.Duration(config.Transport.Dialer.Timeout) * time.Second,
DualStack: config.Transport.Dialer.DualStack,
}
d := &helpers.Dialer{
Dialer: d0,
Resolver: &helpers.Resolver{
LRUCache: lrucache.NewLRUCache(config.Transport.Dialer.DNSCacheSize),
DNSExpiry: time.Duration(config.Transport.Dialer.DNSCacheExpiry) * time.Second,
},
Level: 2,
}
for _, server := range servers {
if server.Host != "" {
d.Resolver.LRUCache.Set(server.URL.Hostname(), server.Host, time.Time{})
}
}
tr := &http.Transport{
Dial: d.Dial,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: false,
ClientSessionCache: tls.NewLRUClientSessionCache(1000),
},
TLSHandshakeTimeout: time.Duration(config.Transport.TLSHandshakeTimeout) * time.Second,
MaxIdleConnsPerHost: config.Transport.MaxIdleConnsPerHost,
}
if config.Transport.Proxy.Enabled {
fixedURL, err := url.Parse(config.Transport.Proxy.URL)
if err != nil {
glog.Fatalf("url.Parse(%#v) error: %s", config.Transport.Proxy.URL, err)
}
switch fixedURL.Scheme {
case "http", "https":
tr.Proxy = http.ProxyURL(fixedURL)
tr.Dial = nil
tr.DialTLS = nil
default:
dialer, err := proxy.FromURL(fixedURL, d, nil)
if err != nil {
glog.Fatalf("proxy.FromURL(%#v) error: %s", fixedURL.String(), err)
}
tr.Dial = dialer.Dial
tr.DialTLS = nil
tr.Proxy = nil
}
}
if tr.TLSClientConfig != nil {
err := http2.ConfigureTransport(tr)
if err != nil {
glog.Warningf("PHP: Error enabling Transport HTTP/2 support: %v", err)
}
}
return &Filter{
Config: *config,
Transport: &Transport{
RoundTripper: tr,
Servers: servers,
},
}, nil
}
func (f *Filter) Response(ctx context.Context, resp *http.Response) (context.Context, *http.Response, error) {
if resp.StatusCode != http.StatusPartialContent || resp.Header.Get("Content-Length") == "" {
return ctx, resp, nil
}
if ok1, ok := filters.Bool(ctx, "autorange.default"); ok && ok1 {
return ctx, resp, nil
}
f1 := filters.GetRoundTripFilter(ctx)
if f1 == nil {
return ctx, resp, nil
}
if _, ok := f.SupportFilters[f1.FilterName()]; !ok {
glog.V(2).Infof("AUTORANGE hit a unsupported filter=%#v", f1)
return ctx, resp, nil
}
parts := strings.Split(resp.Header.Get("Content-Range"), " ")
if len(parts) != 2 || parts[0] != "bytes" {
return ctx, resp, nil
}
parts1 := strings.Split(parts[1], "/")
parts2 := strings.Split(parts1[0], "-")
if len(parts1) != 2 || len(parts2) != 2 {
return ctx, resp, nil
}
var end, length int64
var err error
end, err = strconv.ParseInt(parts2[1], 10, 64)
if err != nil {
return ctx, resp, nil
}
length, err = strconv.ParseInt(parts1[1], 10, 64)
if err != nil {
return ctx, resp, nil
}
glog.V(2).Infof("AUTORANGE respone matched, start rangefetch for %#v", resp.Header.Get("Content-Range"))
resp.ContentLength = length
resp.Header.Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10))
resp.Header.Del("Content-Range")
r, w := AutoPipe(f.Threads)
go func(w *autoPipeWriter, filter filters.RoundTripFilter, req0 *http.Request, start, length int64) {
glog.V(2).Infof("AUTORANGE begin rangefetch for %#v by using %#v", req0.URL.String(), filter.FilterName())
req, err := http.NewRequest(req0.Method, req0.URL.String(), nil)
if err != nil {
glog.Warningf("AUTORANGE http.NewRequest(%#v) error: %#v", req, err)
return
}
for key, values := range req0.Header {
for _, value := range values {
req.Header.Add(key, value)
}
}
if err := w.WaitForReading(); err != nil {
return
}
var index uint32
for {
if w.FatalErr() {
break
}
if start > length-1 {
// tell reader io.EOF
w.Close()
break
}
//FIXME: make this configurable!
if w.Len() > 128*1024*1024 {
time.Sleep(100 * time.Millisecond)
continue
}
//end := start + int64(f.BufSize-1)
end := start + int64(1024<<10-1)
if end > length-1 {
end = length - 1
}
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
_, resp, err := filter.RoundTrip(nil, req)
if err != nil {
glog.Warningf("AUTORANGE %#v.RoundTrip(%v) error: %#v", filter, req, err)
time.Sleep(1 * time.Second)
continue
}
if resp.StatusCode != http.StatusPartialContent {
if resp.StatusCode >= http.StatusBadRequest {
time.Sleep(1 * time.Second)
}
continue
}
w.ThreadHello()
go func(index uint32, resp *http.Response) {
defer resp.Body.Close()
defer w.ThreadBye() // TODO: launch it as soon as iocopy over?
piper := w.NewPiper(index)
_, err := helpers.IoCopy(piper, resp.Body)
if err != nil {
glog.Warningf("AUTORANGE helpers.IoCopy(%#v) error: %#v", resp.Body, err)
piper.EIndex()
}
piper.WClose()
}(index, resp)
start = end + 1
index++
}
}(w, f1, resp.Request, end+1, length)
resp.Body = helpers.NewMultiReadCloser(resp.Body, r)
return ctx, resp, nil
}
func (s *Servers) DecodeResponse(resp *http.Response) (resp1 *http.Response, err error) {
if resp.StatusCode != http.StatusOK {
return resp, nil
}
var hdrLen uint16
if err = binary.Read(resp.Body, binary.BigEndian, &hdrLen); err != nil {
return
}
hdrBuf := make([]byte, hdrLen)
if _, err = io.ReadFull(resp.Body, hdrBuf); err != nil {
return
}
resp1, err = http.ReadResponse(bufio.NewReader(flate.NewReader(bytes.NewReader(hdrBuf))), resp.Request)
if err != nil {
return
}
const cookieKey string = "Set-Cookie"
if cookies, ok := resp1.Header[cookieKey]; ok && len(cookies) == 1 {
parts := strings.Split(cookies[0], ", ")
parts1 := make([]string, 0)
for i := 0; i < len(parts); i++ {
c := parts[i]
if i == 0 || strings.Contains(strings.Split(c, ";")[0], "=") {
parts1 = append(parts1, c)
} else {
parts1[len(parts1)-1] = parts1[len(parts1)-1] + ", " + c
}
}
if len(parts1) > 1 {
glog.Warningf("FetchServer(%+v) is not a goproxy GAE server, please upgrade!", resp.Request.Host)
resp1.Header.Del(cookieKey)
for i := 0; i < len(parts1); i++ {
resp1.Header.Add(cookieKey, parts1[i])
}
}
}
if resp1.StatusCode >= http.StatusBadRequest {
switch {
case resp.Body == nil:
break
case resp1.Body == nil:
resp1.Body = resp.Body
default:
b, _ := ioutil.ReadAll(resp1.Body)
if b != nil && len(b) > 0 {
resp1.Body = helpers.NewMultiReadCloser(bytes.NewReader(b), resp.Body)
} else {
resp1.Body = resp.Body
}
}
} else {
resp1.Body = resp.Body
}
return
}
func (f *Filter) updater() {
glog.V(2).Infof("start updater for %+v, expiry=%s, duration=%s", f.GFWList.URL.String(), f.GFWList.Expiry, f.GFWList.Duration)
ticker := time.Tick(f.GFWList.Duration)
for {
select {
case <-ticker:
glog.V(2).Infof("Begin auto gfwlist(%#v) update...", f.GFWList.URL.String())
h, err := f.Store.HeadObject(f.GFWList.Filename)
if err != nil {
glog.Warningf("stat gfwlist(%#v) err: %v", f.GFWList.Filename, err)
continue
}
lm := h.Get("Last-Modified")
if lm == "" {
glog.Warningf("gfwlist(%#v) header(%#v) does not contains last-modified", f.GFWList.Filename, h)
continue
}
modTime, err := time.Parse(f.Store.DateFormat(), lm)
if err != nil {
glog.Warningf("stat gfwlist(%#v) has parse %#v error: %v", f.GFWList.Filename, lm, err)
continue
}
if time.Now().Sub(modTime) < f.GFWList.Expiry {
continue
}
}
glog.Infof("Downloading %#v", f.GFWList.URL.String())
req, err := http.NewRequest(http.MethodGet, f.GFWList.URL.String(), nil)
if err != nil {
glog.Warningf("NewRequest(%#v) error: %v", f.GFWList.URL.String(), err)
continue
}
resp, err := f.Transport.RoundTrip(req)
if err != nil {
glog.Warningf("%T.RoundTrip(%#v) error: %v", f.Transport, f.GFWList.URL.String(), err.Error())
continue
}
var r io.Reader = resp.Body
switch f.GFWList.Encoding {
case "base64":
r = base64.NewDecoder(base64.StdEncoding, r)
default:
break
}
data, err := ioutil.ReadAll(r)
if err != nil {
glog.Warningf("ioutil.ReadAll(%T) error: %v", r, err)
resp.Body.Close()
continue
}
err = f.Store.DeleteObject(f.GFWList.Filename)
if err != nil {
glog.Warningf("%T.DeleteObject(%#v) error: %v", f.Store, f.GFWList.Filename, err)
continue
}
err = f.Store.PutObject(f.GFWList.Filename, http.Header{}, ioutil.NopCloser(bytes.NewReader(data)))
if err != nil {
glog.Warningf("%T.PutObject(%#v) error: %v", f.Store, f.GFWList.Filename, err)
continue
}
glog.Infof("Update %#v from %#v OK", f.GFWList.Filename, f.GFWList.URL.String())
resp.Body.Close()
}
}
func (h Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
var err error
remoteAddr := req.RemoteAddr
// Prepare filter.Context
ctx := filters.NewContext(req.Context(), h, h.Listener, rw, h.Branding)
req = req.WithContext(ctx)
// Enable transport http proxy
if req.Method != "CONNECT" && !req.URL.IsAbs() {
if req.URL.Scheme == "" {
if req.TLS != nil && req.ProtoMajor == 1 {
req.URL.Scheme = "https"
} else {
req.URL.Scheme = "http"
}
}
if req.TLS != nil {
if req.Host == "" {
if req.URL.Host != "" {
req.Host = req.URL.Host
} else {
req.Host = req.TLS.ServerName
}
}
if req.URL.Host == "" {
if req.Host != "" {
req.URL.Host = req.Host
} else {
req.URL.Host = req.TLS.ServerName
}
}
}
}
// Filter Request
for _, f := range h.RequestFilters {
ctx, req, err = f.Request(ctx, req)
if req == filters.DummyRequest {
return
}
if err != nil {
if err != io.EOF {
glog.Errorf("%s Filter Request %T error: %+v", remoteAddr, f, err)
}
return
}
// Update context for request
req = req.WithContext(ctx)
}
if req.Body != nil {
defer req.Body.Close()
}
// Filter Request -> Response
var resp *http.Response
for _, f := range h.RoundTripFilters {
ctx, resp, err = f.RoundTrip(ctx, req)
if resp == filters.DummyResponse {
return
}
// Unexcepted errors
if err != nil {
filters.SetRoundTripFilter(ctx, f)
glog.Errorf("%s Filter RoundTrip %T error: %+v", remoteAddr, f, err)
http.Error(rw, h.FormatError(ctx, err), http.StatusBadGateway)
return
}
// Update context for request
req = req.WithContext(ctx)
// A roundtrip filter give a response
if resp != nil {
resp.Request = req
filters.SetRoundTripFilter(ctx, f)
break
}
}
// Filter Response
for _, f := range h.ResponseFilters {
if resp == nil || resp == filters.DummyResponse {
return
}
ctx, resp, err = f.Response(ctx, resp)
if err != nil {
glog.Errorln("%s Filter %T Response error: %+v", remoteAddr, f, err)
http.Error(rw, h.FormatError(ctx, err), http.StatusBadGateway)
return
}
// Update context for request
req = req.WithContext(ctx)
}
if resp == nil {
glog.Errorln("%s Handler %#v Response empty response", remoteAddr, h)
http.Error(rw, h.FormatError(ctx, fmt.Errorf("empty response")), http.StatusBadGateway)
return
}
if resp.Header.Get("Content-Length") == "" && resp.ContentLength >= 0 {
resp.Header.Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10))
}
for key, values := range resp.Header {
for _, value := range values {
rw.Header().Add(key, value)
}
}
rw.WriteHeader(resp.StatusCode)
if resp.Body != nil {
defer resp.Body.Close()
n, err := helpers.IOCopy(rw, resp.Body)
if err != nil {
if isClosedConnError(err) {
glog.Infof("IOCopy %#v return %#v %T(%v)", resp.Body, n, err, err)
} else {
glog.Warningf("IOCopy %#v return %#v %T(%v)", resp.Body, n, err, err)
}
}
}
}
func (d *MultiDialer) DialTLS2(network, address string, cfg *tls.Config) (net.Conn, error) {
glog.Warningf("MULTIDIALER DialTLS2(%#v, %#v) with good_addrs=%d, bad_addrs=%d", network, address, d.TLSConnDuration.Len(), d.TLSConnError.Len())
if cfg == nil {
cfg = d.TLSConfig
}
switch network {
case "tcp", "tcp4", "tcp6":
if host, port, err := net.SplitHostPort(address); err == nil {
if alias0, ok := d.Site2Alias.Lookup(host); ok {
alias := alias0.(string)
if hosts, err := d.LookupAlias(alias); err == nil {
var config *tls.Config
isGoogleAddr := false
switch {
case strings.HasPrefix(alias, "google_"):
config = d.GoogleTLSConfig
isGoogleAddr = true
case cfg == nil:
config = &tls.Config{
InsecureSkipVerify: !d.SSLVerify,
ServerName: address,
}
default:
config = cfg
}
glog.V(3).Infof("DialTLS2(%#v, %#v) alais=%#v set tls.Config=%#v", network, address, alias, config)
addrs := make([]string, len(hosts))
for i, host := range hosts {
addrs[i] = net.JoinHostPort(host, port)
}
if d.ForceIPv6 {
network = "tcp6"
}
conn, err := d.dialMultiTLS(network, addrs, config)
if err != nil {
return nil, err
}
if d.SSLVerify && isGoogleAddr {
if tc, ok := conn.(*tls.Conn); ok {
cert := tc.ConnectionState().PeerCertificates[0]
glog.V(3).Infof("MULTIDIALER DialTLS(%#v, %#v) verify cert=%v", network, address, cert.Subject)
if d.GoogleG2KeyID != nil {
if bytes.Compare(cert.AuthorityKeyId, d.GoogleG2KeyID) != 0 {
defer conn.Close()
return nil, fmt.Errorf("Wrong certificate of %s: Issuer=%v, AuthorityKeyId=%#v", conn.RemoteAddr(), cert.Issuer, cert.AuthorityKeyId)
}
} else {
if !strings.HasPrefix(cert.Issuer.CommonName, "Google ") {
defer conn.Close()
return nil, fmt.Errorf("Wrong certificate of %s: Issuer=%v, AuthorityKeyId=%#v", conn.RemoteAddr(), cert.Issuer, cert.AuthorityKeyId)
}
}
}
}
return conn, nil
}
}
}
default:
break
}
return tls.DialWithDialer(&d.Dialer, network, address, d.TLSConfig)
}
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
if !f.SiteMatcher.Match(req.Host) {
return ctx, nil, nil
}
var tr http.RoundTripper = f.GAETransport
if req.URL.Scheme == "http" && f.ForceHTTPSMatcher.Match(req.Host) {
if !strings.HasPrefix(req.Header.Get("Referer"), "https://") {
u := strings.Replace(req.URL.String(), "http://", "https://", 1)
glog.V(2).Infof("GAE FORCEHTTPS get raw url=%v, redirect to %v", req.URL.String(), u)
resp := &http.Response{
StatusCode: http.StatusMovedPermanently,
Header: http.Header{
"Location": []string{u},
},
Request: req,
Close: true,
ContentLength: -1,
}
glog.V(2).Infof("%s \"GAE FORCEHTTPS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
if f.DirectSiteMatcher.Match(req.Host) {
if req.URL.Path == "/url" {
if rawurl := req.URL.Query().Get("url"); rawurl != "" {
if u, err := url.Parse(rawurl); err == nil {
if u.Scheme == "http" && f.ForceHTTPSMatcher.Match(u.Host) {
rawurl = strings.Replace(rawurl, "http://", "https://", 1)
}
}
glog.V(2).Infof("%s \"GAE REDIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, rawurl, req.Proto)
return ctx, &http.Response{
StatusCode: http.StatusFound,
Header: http.Header{
"Location": []string{rawurl},
},
Request: req,
Close: true,
ContentLength: -1,
}, nil
}
}
if req.URL.Scheme != "http" && !f.shouldForceGAE(req) {
tr = f.DirectTransport
if s := req.Header.Get("Connection"); s != "" {
if s1 := strings.ToLower(s); s != s1 {
req.Header.Set("Connection", s1)
}
}
}
}
if tr != f.DirectTransport && req.Method == http.MethodOptions {
if v, ok := f.FakeOptionsMatcher.Lookup(req.Host); ok {
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{},
Request: req,
Close: false,
ContentLength: -1,
}
for _, s := range v.([]string) {
parts := strings.SplitN(s, ":", 2)
if len(parts) == 2 {
resp.Header.Add(parts[0], strings.TrimSpace(parts[1]))
}
}
if origin := req.Header.Get("Origin"); origin != "" {
resp.Header.Set("Access-Control-Allow-Origin", origin)
}
if headers := req.Header.Get("Access-Control-Request-Headers"); headers != "" {
resp.Header.Set("Access-Control-Allow-Headers", headers)
}
glog.V(2).Infof("%s \"GAE FAKEOPTIONS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
prefix := "FETCH"
if tr == f.DirectTransport {
prefix = "DIRECT"
}
resp, err := tr.RoundTrip(req)
if err != nil {
glog.Warningf("%s \"GAE %s %s %s %s\" error: %T(%v)", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, err, err)
if tr == f.DirectTransport {
if ne, ok := err.(interface {
Timeout() bool
}); ok && ne.Timeout() {
// f.MultiDialer.ClearCache()
helpers.TryCloseConnections(tr)
}
}
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
return ctx, nil, err
}
if strings.HasPrefix(resp.Header.Get("Content-Type"), "text/") &&
resp.Header.Get("Content-Encoding") == "" &&
f.ForceDeflateMatcher.Match(req.Host) {
buf := make([]byte, 1024)
n, err := resp.Body.Read(buf)
if err != nil {
defer resp.Body.Close()
return ctx, nil, err
}
buf = buf[:n]
switch {
case helpers.IsGzip(buf):
resp.Header.Set("Content-Encoding", "gzip")
case helpers.IsBinary(buf):
resp.Header.Set("Content-Encoding", "deflate")
}
resp.Body = helpers.NewMultiReadCloser(bytes.NewReader(buf), resp.Body)
}
if resp != nil && resp.Header != nil {
resp.Header.Del("Alt-Svc")
resp.Header.Del("Alternate-Protocol")
}
glog.V(2).Infof("%s \"GAE %s %s %s %s\" %d %s", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, err
}
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
deadline := t.Deadline
for i := 0; i < t.RetryTimes; i++ {
server := t.Servers.PickFetchServer(req, i)
req1, err := t.Servers.EncodeRequest(req, server, deadline)
if err != nil {
return nil, fmt.Errorf("GAE EncodeRequest: %s", err.Error())
}
resp, err := t.RoundTripper.RoundTrip(req1)
if err != nil {
isTimeoutError := false
if ne, ok := err.(interface {
Timeout() bool
}); ok && ne.Timeout() {
isTimeoutError = true
}
if ne, ok := err.(*net.OpError); ok && ne.Op == "read" {
isTimeoutError = true
}
if isTimeoutError {
glog.Warningf("GAE: \"%s %s\" timeout: %v, helpers.CloseConnections(%T)", req.Method, req.URL.String(), err, t.RoundTripper)
helpers.CloseConnections(t.RoundTripper)
}
if i == t.RetryTimes-1 {
return nil, err
} else {
glog.Warningf("GAE: request \"%s\" error: %T(%v), retry...", req.URL.String(), err, err)
continue
}
}
if resp.StatusCode != http.StatusOK {
if i == t.RetryTimes-1 {
return resp, nil
}
switch resp.StatusCode {
case http.StatusServiceUnavailable:
glog.Warningf("GAE: %s over qouta, try switch to next appid...", server.Host)
t.Servers.ToggleBadServer(server)
time.Sleep(t.RetryDelay)
continue
case http.StatusFound,
http.StatusBadGateway,
http.StatusNotFound,
http.StatusMethodNotAllowed:
if t.MultiDialer != nil {
if addr, err := helpers.ReflectRemoteAddrFromResponse(resp); err == nil {
if ip, _, err := net.SplitHostPort(addr); err == nil {
duration := 8 * time.Hour
glog.Warningf("GAE: %s StatusCode is %d, not a gws/gvs ip, add to blacklist for %v", ip, resp.StatusCode, duration)
t.MultiDialer.IPBlackList.Set(ip, struct{}{}, time.Now().Add(duration))
}
if !helpers.CloseConnectionByRemoteAddr(t.RoundTripper, addr) {
glog.Warningf("GAE: CloseConnectionByRemoteAddr(%T, %#v) failed.", t.RoundTripper, addr)
}
}
}
continue
default:
return resp, nil
}
}
resp1, err := t.Servers.DecodeResponse(resp)
if err != nil {
return nil, err
}
if resp1 != nil {
resp1.Request = req
}
if i == t.RetryTimes-1 {
return resp, err
}
switch resp1.StatusCode {
case http.StatusBadGateway:
body, err := ioutil.ReadAll(resp1.Body)
if err != nil {
resp1.Body.Close()
return nil, err
}
resp1.Body.Close()
switch {
case bytes.Contains(body, []byte("DEADLINE_EXCEEDED")):
//FIXME: deadline += 10 * time.Second
glog.Warningf("GAE: %s urlfetch %#v get DEADLINE_EXCEEDED, retry with deadline=%s...", req1.Host, req.URL.String(), deadline)
continue
case bytes.Contains(body, []byte("ver quota")):
glog.Warningf("GAE: %s urlfetch %#v get over quota, retry...", req1.Host, req.URL.String())
time.Sleep(t.RetryDelay)
continue
case bytes.Contains(body, []byte("urlfetch: CLOSED")):
glog.Warningf("GAE: %s urlfetch %#v get urlfetch: CLOSED, retry...", req1.Host, req.URL.String())
time.Sleep(t.RetryDelay)
continue
default:
resp1.Body = ioutil.NopCloser(bytes.NewReader(body))
return resp1, nil
}
default:
return resp1, nil
}
}
return nil, fmt.Errorf("GAE: cannot reach here with %#v", req)
}
func NewRootCA(name string, vaildFor time.Duration, rsaBits int, certDir string, portable bool) (*RootCA, error) {
keyFile := name + ".key"
certFile := name + ".crt"
var store storage.Store
if portable {
exe, err := os.Executable()
if err != nil {
glog.Fatalf("os.Executable() error: %+v", err)
}
store = &storage.FileStore{filepath.Dir(exe)}
} else {
store = &storage.FileStore{"."}
}
rootCA := &RootCA{
store: store,
name: name,
keyFile: keyFile,
certFile: certFile,
rsaBits: rsaBits,
certDir: certDir,
mu: new(sync.Mutex),
}
if storage.NotExist(store, certFile) {
glog.Infof("Generating RootCA for %s/%s", keyFile, certFile)
template := x509.Certificate{
IsCA: true,
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
CommonName: name,
Country: []string{"US"},
Province: []string{"California"},
Locality: []string{"Los Angeles"},
Organization: []string{name},
ExtraNames: []pkix.AttributeTypeAndValue{
{
Type: []int{2, 5, 4, 42},
Value: name,
},
},
},
DNSNames: []string{name},
NotBefore: time.Now().Add(-time.Duration(30 * 24 * time.Hour)),
NotAfter: time.Now().Add(vaildFor),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
BasicConstraintsValid: true,
// AuthorityKeyId: sha1.New().Sum([]byte("phuslu")),
// SubjectKeyId: sha1.New().Sum([]byte("phuslu")),
}
priv, err := rsa.GenerateKey(rand.Reader, rsaBits)
if err != nil {
return nil, err
}
derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
if err != nil {
return nil, err
}
ca, err := x509.ParseCertificate(derBytes)
if err != nil {
return nil, err
}
rootCA.ca = ca
rootCA.priv = priv
rootCA.derBytes = derBytes
keypem := &pem.Block{Type: "PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootCA.priv)}
rc := ioutil.NopCloser(bytes.NewReader(pem.EncodeToMemory(keypem)))
if _, err = store.Put(keyFile, http.Header{}, rc); err != nil {
return nil, err
}
certpem := &pem.Block{Type: "CERTIFICATE", Bytes: rootCA.derBytes}
rc = ioutil.NopCloser(bytes.NewReader(pem.EncodeToMemory(certpem)))
if _, err = store.Put(certFile, http.Header{}, rc); err != nil {
return nil, err
}
} else {
for _, name := range []string{keyFile, certFile} {
resp, err := store.Get(name)
if err != nil {
return nil, err
}
data, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return nil, err
}
var b *pem.Block
for {
b, data = pem.Decode(data)
if b == nil {
break
}
switch b.Type {
case "CERTIFICATE":
rootCA.derBytes = b.Bytes
ca, err := x509.ParseCertificate(rootCA.derBytes)
if err != nil {
return nil, err
}
rootCA.ca = ca
case "PRIVATE KEY", "PRIVATE RSA KEY":
priv, err := x509.ParsePKCS1PrivateKey(b.Bytes)
if err != nil {
return nil, err
}
rootCA.priv = priv
}
}
}
}
switch runtime.GOOS {
case "windows", "darwin":
if _, err := rootCA.ca.Verify(x509.VerifyOptions{}); err != nil {
glog.Warningf("Verify RootCA(%#v) error: %v, try import to system root", name, err)
if err = helpers.RemoveCAFromSystemRoot(rootCA.name); err != nil {
glog.Errorf("Remove Old RootCA(%#v) error: %v", name, err)
}
if err = helpers.ImportCAToSystemRoot(rootCA.ca); err != nil {
glog.Errorf("Import RootCA(%#v) error: %v", name, err)
} else {
glog.Infof("Import RootCA(%s) OK", certFile)
}
if fs, err := store.List(certDir); err == nil {
for _, f := range fs {
if _, err = store.Delete(f); err != nil {
glog.Errorf("%T.Delete(%#v) error: %v", store, f, err)
}
}
}
}
}
if fs, ok := store.(*storage.FileStore); ok {
if storage.NotExist(store, certDir) {
if err := os.Mkdir(filepath.Join(fs.Dirname, certDir), 0777); err != nil {
return nil, err
}
}
}
return rootCA, nil
}
func NewFilter(config *Config) (filters.Filter, error) {
servers := make([]Server, 0)
for _, s := range config.Servers {
u, err := url.Parse(s.URL)
if err != nil {
return nil, err
}
server := Server{
URL: u,
Password: s.Password,
SSLVerify: s.SSLVerify,
Host: s.Host,
}
servers = append(servers, server)
}
d := &dialer.Dialer{
Dialer: net.Dialer{
KeepAlive: time.Duration(config.Transport.Dialer.KeepAlive) * time.Second,
Timeout: time.Duration(config.Transport.Dialer.Timeout) * time.Second,
DualStack: config.Transport.Dialer.DualStack,
},
RetryTimes: config.Transport.Dialer.RetryTimes,
RetryDelay: time.Duration(config.Transport.Dialer.RetryDelay*1000) * time.Second,
DNSCache: lrucache.NewLRUCache(config.Transport.Dialer.DNSCacheSize),
DNSCacheExpiry: time.Duration(config.Transport.Dialer.DNSCacheExpiry) * time.Second,
LoopbackAddrs: nil,
Level: 2,
}
for _, server := range servers {
if server.Host != "" {
host := server.URL.Host
if _, _, err := net.SplitHostPort(host); err != nil {
if server.URL.Scheme == "https" {
host += ":443"
} else {
host += ":80"
}
}
host1 := server.Host
if _, _, err := net.SplitHostPort(host1); err != nil {
if server.URL.Scheme == "https" {
host1 += ":443"
} else {
host1 += ":80"
}
}
d.DNSCache.Set(host, host1, time.Time{})
}
}
tr := &http.Transport{
Dial: d.Dial,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: false,
ClientSessionCache: tls.NewLRUClientSessionCache(1000),
},
TLSHandshakeTimeout: time.Duration(config.Transport.TLSHandshakeTimeout) * time.Second,
MaxIdleConnsPerHost: config.Transport.MaxIdleConnsPerHost,
}
if tr.TLSClientConfig != nil {
err := http2.ConfigureTransport(tr)
if err != nil {
glog.Warningf("PHP: Error enabling Transport HTTP/2 support: %v", err)
}
}
return &Filter{
Config: *config,
Transport: &Transport{
RoundTripper: tr,
Servers: servers,
},
Sites: helpers.NewHostMatcher(config.Sites),
}, nil
}
func NewRootCA(name string, vaildFor time.Duration, rsaBits int, certDir string, portable bool) (*RootCA, error) {
keyFile := name + ".key"
certFile := name + ".crt"
if portable {
rootdir := filepath.Dir(os.Args[0])
keyFile = filepath.Join(rootdir, keyFile)
certFile = filepath.Join(rootdir, certFile)
certDir = filepath.Join(rootdir, certDir)
}
rootCA := &RootCA{
name: name,
keyFile: keyFile,
certFile: certFile,
rsaBits: rsaBits,
certDir: certDir,
mu: new(sync.Mutex),
}
if _, err := os.Stat(certFile); os.IsNotExist(err) {
glog.Infof("Generating RootCA for %s", certFile)
template := x509.Certificate{
IsCA: true,
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
CommonName: name,
Organization: []string{name},
},
NotBefore: time.Now().Add(-time.Duration(30 * 24 * time.Hour)),
NotAfter: time.Now().Add(vaildFor),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
BasicConstraintsValid: true,
}
priv, err := rsa.GenerateKey(rand.Reader, rsaBits)
if err != nil {
return nil, err
}
derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
if err != nil {
return nil, err
}
ca, err := x509.ParseCertificate(derBytes)
if err != nil {
return nil, err
}
rootCA.ca = ca
rootCA.priv = priv
rootCA.derBytes = derBytes
keypem := &pem.Block{Type: "PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootCA.priv)}
if err = ioutil.WriteFile(keyFile, pem.EncodeToMemory(keypem), 0755); err != nil {
return nil, err
}
certpem := &pem.Block{Type: "CERTIFICATE", Bytes: rootCA.derBytes}
if err = ioutil.WriteFile(certFile, pem.EncodeToMemory(certpem), 0755); err != nil {
return nil, err
}
} else {
data, err := ioutil.ReadFile(keyFile)
if err != nil {
return nil, err
}
var b *pem.Block
for {
b, data = pem.Decode(data)
if b == nil {
break
}
if b.Type == "CERTIFICATE" {
rootCA.derBytes = b.Bytes
ca, err := x509.ParseCertificate(rootCA.derBytes)
if err != nil {
return nil, err
}
rootCA.ca = ca
} else if b.Type == "PRIVATE KEY" {
priv, err := x509.ParsePKCS1PrivateKey(b.Bytes)
if err != nil {
return nil, err
}
rootCA.priv = priv
}
}
data, err = ioutil.ReadFile(certFile)
if err != nil {
return nil, err
}
for {
b, data = pem.Decode(data)
if b == nil {
break
}
if b.Type == "CERTIFICATE" {
rootCA.derBytes = b.Bytes
ca, err := x509.ParseCertificate(rootCA.derBytes)
if err != nil {
return nil, err
}
rootCA.ca = ca
} else if b.Type == "PRIVATE KEY" {
priv, err := x509.ParsePKCS1PrivateKey(b.Bytes)
if err != nil {
return nil, err
}
rootCA.priv = priv
}
}
}
switch runtime.GOOS {
case "windows", "darwin":
if _, err := rootCA.ca.Verify(x509.VerifyOptions{}); err != nil {
glog.Warningf("Verify RootCA(%#v) error: %v, try import to system root", name, err)
if err = helpers.RemoveCAFromSystemRoot(rootCA.name); err != nil {
glog.Errorf("Remove Old RootCA(%#v) error: %v", name, err)
}
if err = helpers.ImportCAToSystemRoot(rootCA.ca); err != nil {
glog.Errorf("Import RootCA(%#v) error: %v", name, err)
} else {
glog.Infof("Import RootCA(%s) OK", certFile)
}
if fis, err := ioutil.ReadDir(certDir); err == nil {
for _, fi := range fis {
if err = os.Remove(certDir + "/" + fi.Name()); err != nil {
glog.Errorf("Remove(%#v) error: %v", fi.Name(), err)
}
}
}
}
}
if _, err := os.Stat(certDir); os.IsNotExist(err) {
if err = os.Mkdir(certDir, 0755); err != nil {
return nil, err
}
}
return rootCA, nil
}
