func (f *Filter) Request(ctx context.Context, req *http.Request) (context.Context, *http.Request, error) {
if req.Method != http.MethodGet || strings.Contains(req.URL.RawQuery, "range=") {
return ctx, req, nil
}
if r := req.Header.Get("Range"); r == "" {
switch {
case f.SiteMatcher.Match(req.Host):
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", 0, f.MaxSize))
glog.V(2).Infof("AUTORANGE Sites rule matched, add %s for\"%s\"", req.Header.Get("Range"), req.URL.String())
ctx = filters.WithBool(ctx, "autorange.site", true)
default:
glog.V(3).Infof("AUTORANGE ignore preserved empty range for %#v", req.URL)
}
} else {
ctx = filters.WithBool(ctx, "autorange.default", true)
parts := strings.Split(r, "=")
switch parts[0] {
case "bytes":
parts1 := strings.Split(parts[1], "-")
if start, err := strconv.Atoi(parts1[0]); err == nil {
if end, err := strconv.Atoi(parts1[1]); err != nil || end-start > f.MaxSize {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, start+f.MaxSize))
glog.V(2).Infof("AUTORANGE Default rule matched, change %s to %s for\"%s\"", r, req.Header.Get("Range"), req.URL.String())
}
}
default:
glog.Warningf("AUTORANGE Default rule matched, but cannot support %#v range for \"%s\"", r, req.URL.String())
}
}
return ctx, req, nil
}
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
if f.SiteFiltersEnabled {
if f1, ok := f.SiteFiltersRules.Lookup(req.Host); ok {
glog.V(2).Infof("%s \"AUTOPROXY SiteFilters %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
return f1.(filters.RoundTripFilter).RoundTrip(ctx, req)
}
}
if f.RegionFiltersEnabled {
//TODO
}
if req.URL.Host == "" && req.RequestURI[0] == '/' && f.IndexFilesEnabled {
if _, ok := f.IndexFiles[req.URL.Path[1:]]; ok || req.URL.Path == "/" {
if f.GFWListEnabled && strings.HasSuffix(req.URL.Path, ".pac") {
glog.V(2).Infof("%s \"AUTOPROXY ProxyPac %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
return f.ProxyPacRoundTrip(ctx, req)
} else {
glog.V(2).Infof("%s \"AUTOPROXY IndexFiles %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
return f.IndexFilesRoundTrip(ctx, req)
}
}
}
return ctx, nil, nil
}
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
switch req.Method {
case "CONNECT":
glog.V(2).Infof("%s \"DIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, req.Host, req.Proto)
rconn, err := f.transport.Dial("tcp", req.Host)
if err != nil {
return ctx, nil, err
}
rw := filters.GetResponseWriter(ctx)
hijacker, ok := rw.(http.Hijacker)
if !ok {
return ctx, nil, fmt.Errorf("http.ResponseWriter(%#v) does not implments http.Hijacker", rw)
}
flusher, ok := rw.(http.Flusher)
if !ok {
return ctx, nil, fmt.Errorf("http.ResponseWriter(%#v) does not implments http.Flusher", rw)
}
rw.WriteHeader(http.StatusOK)
flusher.Flush()
lconn, _, err := hijacker.Hijack()
if err != nil {
return ctx, nil, fmt.Errorf("%#v.Hijack() error: %v", hijacker, err)
}
defer lconn.Close()
go helpers.IoCopy(rconn, lconn)
helpers.IoCopy(lconn, rconn)
filters.SetHijacked(ctx, true)
return ctx, nil, nil
default:
helpers.FixRequestURL(req)
resp, err := f.transport.RoundTrip(req)
if err != nil {
glog.Errorf("%s \"DIRECT %s %s %s\" error: %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, err)
data := err.Error()
resp = &http.Response{
StatusCode: http.StatusBadGateway,
Header: http.Header{},
Request: req,
Close: true,
ContentLength: int64(len(data)),
Body: ioutil.NopCloser(bytes.NewReader([]byte(data))),
}
err = nil
} else {
if req.RemoteAddr != "" {
glog.V(2).Infof("%s \"DIRECT %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
}
}
return ctx, resp, err
}
}
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
if ip, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
if _, ok := f.WhiteList[ip]; ok {
return ctx, nil, nil
}
}
if auth := filters.String(ctx, authHeader); auth != "" {
if _, ok := f.ByPassHeaders.Get(auth); ok {
glog.V(3).Infof("auth filter hit bypass cache %#v", auth)
return ctx, nil, nil
}
parts := strings.SplitN(auth, " ", 2)
if len(parts) == 2 {
switch parts[0] {
case "Basic":
if userpass, err := base64.StdEncoding.DecodeString(parts[1]); err == nil {
parts := strings.Split(string(userpass), ":")
user := parts[0]
pass := parts[1]
pass1, ok := f.Basic[user]
if ok && pass == pass1 {
f.ByPassHeaders.Set(auth, struct{}{}, time.Now().Add(time.Hour))
return ctx, nil, nil
}
}
default:
glog.Errorf("Unrecognized auth type: %#v", parts[0])
break
}
}
}
glog.V(1).Infof("UnAuthenticated URL %v from %#v", req.URL.String(), req.RemoteAddr)
noAuthResponse := &http.Response{
Status: "407 Proxy Authentication Required",
StatusCode: http.StatusProxyAuthRequired,
Proto: "HTTP/1.1",
ProtoMajor: 1,
ProtoMinor: 1,
Header: http.Header{},
Request: req,
Close: true,
ContentLength: -1,
}
return ctx, noAuthResponse, nil
}
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
switch req.Method {
case "CONNECT":
glog.V(2).Infof("%s \"DIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, req.Host, req.Proto)
rconn, err := f.transport.Dial("tcp", req.Host)
if err != nil {
return ctx, nil, err
}
rw := filters.GetResponseWriter(ctx)
hijacker, ok := rw.(http.Hijacker)
if !ok {
return ctx, nil, fmt.Errorf("http.ResponseWriter(%#v) does not implments http.Hijacker", rw)
}
flusher, ok := rw.(http.Flusher)
if !ok {
return ctx, nil, fmt.Errorf("http.ResponseWriter(%#v) does not implments http.Flusher", rw)
}
rw.WriteHeader(http.StatusOK)
flusher.Flush()
lconn, _, err := hijacker.Hijack()
if err != nil {
return ctx, nil, fmt.Errorf("%#v.Hijack() error: %v", hijacker, err)
}
defer lconn.Close()
go helpers.IOCopy(rconn, lconn)
helpers.IOCopy(lconn, rconn)
return ctx, filters.DummyResponse, nil
default:
helpers.FixRequestURL(req)
helpers.FixRequestHeader(req)
resp, err := f.transport.RoundTrip(req)
if err != nil {
return ctx, nil, err
}
if req.RemoteAddr != "" {
glog.V(2).Infof("%s \"DIRECT %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
}
return ctx, resp, err
}
}
func (c *RootCA) Issue(commonName string, vaildFor time.Duration, rsaBits int) (*tls.Certificate, error) {
certFile := c.toFilename(commonName, ".crt")
if storage.NotExist(c.store, certFile) {
glog.V(2).Infof("Issue %s certificate for %#v...", c.name, commonName)
c.mu.Lock()
defer c.mu.Unlock()
if storage.NotExist(c.store, certFile) {
if err := c.issue(commonName, vaildFor, rsaBits); err != nil {
return nil, err
}
}
}
resp, err := c.store.Get(certFile)
if err != nil {
return nil, err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
tlsCert, err := tls.X509KeyPair(data, data)
if err != nil {
return nil, err
}
return &tlsCert, nil
}
func (f *Filter) Request(ctx context.Context, req *http.Request) (context.Context, *http.Request, error) {
if f.UserAgentEnabled {
glog.V(3).Infof("REWRITE %#v User-Agent=%#v", req.URL.String(), f.UserAgentValue)
req.Header.Set("User-Agent", f.UserAgentValue)
}
if f.HostEnabled {
if host := req.Header.Get(f.HostRewriteBy); host != "" {
glog.V(3).Infof("REWRITE %#v Host=%#v", req.URL.String(), host)
req.Host = host
req.Header.Set("Host", req.Host)
req.Header.Del(f.HostRewriteBy)
}
}
return ctx, req, nil
}
func (f *Filter) Request(ctx context.Context, req *http.Request) (context.Context, *http.Request, error) {
if f.UserAgentEnabled {
glog.V(3).Infof("REWRITE %#v User-Agent=%#v", req.URL.String(), f.UserAgentValue)
req.Header.Set("User-Agent", f.UserAgentValue)
}
return ctx, req, nil
}
func (d *MultiDialer) ExpandAlias(alias string) error {
names, ok := d.HostMap[alias]
if !ok {
return fmt.Errorf("alias %#v not exists", alias)
}
expire := time.Now().Add(24 * time.Hour)
for _, name := range names {
seen := make(map[string]struct{}, 0)
for _, dnsserver := range d.DNSServers {
var addrs []string
var err error
if net.ParseIP(name) != nil {
addrs = []string{name}
expire = time.Time{}
} else if addrs, err = d.lookupHost2(name, dnsserver); err != nil {
glog.V(2).Infof("lookupHost2(%#v) error: %s", name, err)
continue
}
glog.V(2).Infof("ExpandList(%#v) %#v return %v", name, dnsserver, addrs)
for _, addr := range addrs {
seen[addr] = struct{}{}
}
}
if len(seen) == 0 {
continue
}
if addrs, ok := d.DNSCache.Get(name); ok {
addrs1 := addrs.([]string)
for _, addr := range addrs1 {
seen[addr] = struct{}{}
}
}
addrs := make([]string, 0)
for addr, _ := range seen {
addrs = append(addrs, addr)
}
d.DNSCache.Set(name, addrs, expire)
}
return nil
}
func (f *Filter) Request(ctx context.Context, req *http.Request) (context.Context, *http.Request, error) {
if strings.HasPrefix(req.RequestURI, "/") {
return ctx, req, nil
}
host := req.Host
if h, _, err := net.SplitHostPort(req.Host); err == nil {
host = h
}
if f.SiteFiltersEnabled {
if f1, ok := f.SiteFiltersRules.Lookup(host); ok {
glog.V(2).Infof("%s \"AUTOPROXY SiteFilters %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
filters.SetRoundTripFilter(ctx, f1.(filters.RoundTripFilter))
return ctx, req, nil
}
}
if f.RegionFiltersEnabled {
if f1, ok := f.RegionFilterCache.Get(host); ok {
filters.SetRoundTripFilter(ctx, f1.(filters.RoundTripFilter))
} else if ips, err := net.LookupHost(host); err == nil {
ip := ips[0]
if strings.Contains(ip, ":") {
if f1, ok := f.RegionFiltersRules["ipv6"]; ok {
glog.V(2).Infof("%s \"AUTOPROXY RegionFilters IPv6 %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
f.RegionFilterCache.Set(host, f1, time.Now().Add(time.Hour))
filters.SetRoundTripFilter(ctx, f1)
}
} else if country, err := f.FindCountryByIP(ip); err == nil {
if f1, ok := f.RegionFiltersRules[country]; ok {
glog.V(2).Infof("%s \"AUTOPROXY RegionFilters %s %s %s %s\" with %T", req.RemoteAddr, country, req.Method, req.URL.String(), req.Proto, f1)
f.RegionFilterCache.Set(host, f1, time.Now().Add(time.Hour))
filters.SetRoundTripFilter(ctx, f1)
} else if f1, ok := f.RegionFiltersRules["default"]; ok {
glog.V(2).Infof("%s \"AUTOPROXY RegionFilters Default %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
f.RegionFilterCache.Set(host, f1, time.Now().Add(time.Hour))
filters.SetRoundTripFilter(ctx, f1)
}
}
}
}
return ctx, req, nil
}
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
resp, err := f.Transport.RoundTrip(req)
if err != nil {
return ctx, nil, err
} else {
glog.V(2).Infof("%s \"PHP %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
}
return ctx, resp, nil
}
func (f *Filter) Response(ctx context.Context, resp *http.Response) (context.Context, *http.Response, error) {
if f.Rate > 0 && resp.ContentLength > f.Threshold {
glog.V(2).Infof("RateLimit %#v rate to %#v", resp.Request.URL.String(), f.Rate)
resp.Body = NewRateLimitReader(resp.Body, f.Rate, f.Capacity)
}
return ctx, resp, nil
}
func (d *MultiDialer) LookupAlias(alias string) (addrs []string, err error) {
names, ok := d.HostMap[alias]
if !ok {
return nil, fmt.Errorf("alias %#v not exists", alias)
}
seen := make(map[string]struct{}, 0)
expiry := time.Now().Add(d.DNSCacheExpiry)
for _, name := range names {
var addrs0 []string
if net.ParseIP(name) != nil {
addrs0 = []string{name}
expiry = time.Time{}
} else if addrs1, ok := d.DNSCache.Get(name); ok {
addrs0 = addrs1.([]string)
} else {
if d.ForceIPv6 {
addrs0, err = d.LookupHost2(name, d.DNSServers[0])
if err != nil {
glog.Warningf("LookupHost2(%#v, %#v) error: %s", name, d.DNSServers[0], err)
addrs0 = []string{}
}
} else {
addrs0, err = d.LookupHost(name)
if err != nil {
glog.Warningf("LookupHost(%#v) error: %s", name, err)
addrs0 = []string{}
}
}
glog.V(2).Infof("LookupHost(%#v) return %v", name, addrs0)
d.DNSCache.Set(name, addrs0, expiry)
}
for _, addr := range addrs0 {
seen[addr] = struct{}{}
}
}
if len(seen) == 0 {
return nil, err
}
addrs = make([]string, 0)
for addr, _ := range seen {
if _, ok := d.IPBlackList.GetQuiet(addr); ok {
continue
}
addrs = append(addrs, addr)
}
if len(addrs) == 0 {
glog.Errorf("MULTIDIALER: LookupAlias(%#v) have no good ip addrs", alias)
return nil, fmt.Errorf("MULTIDIALER: LookupAlias(%#v) have no good ip addrs", alias)
}
return addrs, nil
}
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
if f := filters.GetRoundTripFilter(ctx); f != nil {
return f.RoundTrip(ctx, req)
}
if req.URL.Host == "" && req.RequestURI[0] == '/' && f.IndexFilesEnabled {
if _, ok := f.IndexFiles[req.URL.Path[1:]]; ok || req.URL.Path == "/" {
if f.GFWListEnabled && strings.HasSuffix(req.URL.Path, ".pac") {
glog.V(2).Infof("%s \"AUTOPROXY ProxyPac %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
return f.ProxyPacRoundTrip(ctx, req)
} else {
glog.V(2).Infof("%s \"AUTOPROXY IndexFiles %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
return f.IndexFilesRoundTrip(ctx, req)
}
}
}
return ctx, nil, nil
}
func (d *MultiDialer) dialMulti(network string, addrs []string) (net.Conn, error) {
glog.V(3).Infof("dialMulti(%v, %v)", network, addrs)
type racer struct {
c net.Conn
e error
}
length := len(addrs)
if d.Level < length {
length = d.Level
}
addrs = pickupAddrs(addrs, length, d.TCPConnDuration, d.TCPConnError)
lane := make(chan racer, length)
for _, addr := range addrs {
go func(addr string, c chan<- racer) {
start := time.Now()
conn, err := d.Dialer.Dial(network, addr)
end := time.Now()
if err == nil {
if d.TCPConnReadBuffer > 0 {
if tc, ok := conn.(*net.TCPConn); ok {
tc.SetReadBuffer(d.TCPConnReadBuffer)
}
}
d.TCPConnDuration.Set(addr, end.Sub(start), end.Add(d.ConnExpiry))
} else {
d.TCPConnDuration.Del(addr)
d.TCPConnError.Set(addr, err, end.Add(d.ConnExpiry))
}
lane <- racer{conn, err}
}(addr, lane)
}
var r racer
for i := 0; i < length; i++ {
r = <-lane
if r.e == nil {
go func(count int) {
var r1 racer
for ; count > 0; count-- {
r1 = <-lane
if r1.c != nil {
r1.c.Close()
}
}
}(length - 1 - i)
return r.c, nil
}
}
return nil, r.e
}
func (r *Resolver) LookupIP(name string) ([]net.IP, error) {
if r.LRUCache != nil {
if v, ok := r.LRUCache.Get(name); ok {
switch v.(type) {
case []net.IP:
return v.([]net.IP), nil
case string:
return r.LookupIP(v.(string))
default:
return nil, fmt.Errorf("LookupIP: cannot convert %T(%+v) to []net.IP", v, v)
}
}
}
if ip := net.ParseIP(name); ip != nil {
ips := []net.IP{ip}
if r.LRUCache != nil {
r.LRUCache.Set(name, ips, time.Time{})
}
return ips, nil
}
lookupIP := r.lookupIP1
if r.DNSServer != nil {
lookupIP = r.lookupIP2
}
ips, err := lookupIP(name)
if err == nil {
if r.BlackList != nil {
ips1 := ips[:0]
for _, ip := range ips {
if _, ok := r.BlackList.GetQuiet(ip.String()); !ok {
ips1 = append(ips1, ip)
}
}
ips = ips1
}
if r.LRUCache != nil && len(ips) > 0 {
if r.DNSExpiry == 0 {
r.LRUCache.Set(name, ips, time.Time{})
} else {
r.LRUCache.Set(name, ips, time.Now().Add(r.DNSExpiry))
}
}
}
glog.V(2).Infof("LookupIP(%#v) return %+v, err=%+v", name, ips, err)
return ips, err
}
func (f *Filter) Request(ctx context.Context, req *http.Request) (context.Context, *http.Request, error) {
if strings.HasPrefix(req.RequestURI, "/") {
return ctx, req, nil
}
host := helpers.GetHostName(req)
if f.BlackListEnabled {
if f.BlackListSiteMatcher.Match(host) {
glog.V(2).Infof("%s \"AUTOPROXY BlackList %s %s %s\"", req.RemoteAddr, req.Method, req.URL.String(), req.Proto)
return ctx, filters.DummyRequest, nil
}
}
if f.SiteFiltersEnabled {
if f1, ok := f.SiteFiltersRules.Lookup(host); ok {
glog.V(2).Infof("%s \"AUTOPROXY SiteFilters %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
filters.SetRoundTripFilter(ctx, f1.(filters.RoundTripFilter))
return ctx, req, nil
}
}
if f.RegionFiltersEnabled {
if f1, ok := f.RegionFilterCache.Get(host); ok {
if f1 != nil {
filters.SetRoundTripFilter(ctx, f1.(filters.RoundTripFilter))
}
} else if ips, err := f.RegionResolver.LookupIP(host); err == nil && len(ips) > 0 {
ip := ips[0]
if ip.IsLoopback() && !(strings.Contains(host, ".local") || strings.Contains(host, "localhost.")) {
glog.V(2).Infof("%s \"AUTOPROXY RegionFilters BYPASS Loopback %s %s %s\" with nil", req.RemoteAddr, req.Method, req.URL.String(), req.Proto)
f.RegionFilterCache.Set(host, nil, time.Now().Add(time.Hour))
} else if ip.To4() == nil {
if f1, ok := f.RegionFiltersRules["ipv6"]; ok {
glog.V(2).Infof("%s \"AUTOPROXY RegionFilters IPv6 %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
f.RegionFilterCache.Set(host, f1, time.Now().Add(time.Hour))
filters.SetRoundTripFilter(ctx, f1)
}
} else if country, err := f.FindCountryByIP(ip.String()); err == nil {
if f1, ok := f.RegionFiltersRules[country]; ok {
glog.V(2).Infof("%s \"AUTOPROXY RegionFilters %s %s %s %s\" with %T", req.RemoteAddr, country, req.Method, req.URL.String(), req.Proto, f1)
f.RegionFilterCache.Set(host, f1, time.Now().Add(time.Hour))
filters.SetRoundTripFilter(ctx, f1)
} else if f1, ok := f.RegionFiltersRules["default"]; ok {
glog.V(2).Infof("%s \"AUTOPROXY RegionFilters Default %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
f.RegionFilterCache.Set(host, f1, time.Now().Add(time.Hour))
filters.SetRoundTripFilter(ctx, f1)
} else {
f.RegionFilterCache.Set(host, nil, time.Now().Add(time.Hour))
}
}
}
}
return ctx, req, nil
}
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
if f := filters.GetRoundTripFilter(ctx); f != nil {
return f.RoundTrip(ctx, req)
}
switch {
case f.SiteFiltersEnabled && req.URL.Scheme == "https":
if f1, ok := f.SiteFiltersRules.Lookup(helpers.GetHostName(req)); ok && f1 != nil {
return f1.(filters.RoundTripFilter).RoundTrip(ctx, req)
}
case f.RegionFiltersEnabled && req.URL.Scheme == "https":
if f1, ok := f.RegionFilterCache.Get(helpers.GetHostName(req)); ok && f1 != nil {
return f1.(filters.RoundTripFilter).RoundTrip(ctx, req)
}
}
if req.URL.Host == "" && req.RequestURI[0] == '/' && f.IndexFilesEnabled {
if _, ok := f.IndexFilesSet[req.URL.Path[1:]]; ok || req.URL.Path == "/" {
switch {
case f.GFWListEnabled && strings.HasSuffix(req.URL.Path, ".pac"):
glog.V(2).Infof("%s \"AUTOPROXY ProxyPac %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
return f.ProxyPacRoundTrip(ctx, req)
case f.MobileConfigEnabled && strings.HasSuffix(req.URL.Path, ".mobileconfig"):
glog.V(2).Infof("%s \"AUTOPROXY ProxyMobileConfig %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
return f.ProxyMobileConfigRoundTrip(ctx, req)
case f.IPHTMLEnabled && req.URL.Path == "/ip.html":
glog.V(2).Infof("%s \"AUTOPROXY IPHTML %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
return f.IPHTMLRoundTrip(ctx, req)
default:
glog.V(2).Infof("%s \"AUTOPROXY IndexFiles %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
return f.IndexFilesRoundTrip(ctx, req)
}
}
}
return ctx, nil, nil
}
func (d *Dialer) Dial(network, address string) (conn net.Conn, err error) {
glog.V(3).Infof("Dail(%#v, %#v)", network, address)
switch network {
case "tcp", "tcp4", "tcp6":
if d.Resolver != nil {
if host, port, err := net.SplitHostPort(address); err == nil {
if ips, err := d.Resolver.LookupIP(host); err == nil {
if len(ips) == 0 {
return nil, net.InvalidAddrError(fmt.Sprintf("Invaid DNS Record: %s", address))
}
return d.dialMulti(network, address, ips, port)
}
}
}
}
return d.Dialer.Dial(network, address)
}
func (c *RootCA) Issue(commonName string, vaildFor time.Duration, rsaBits int) (*tls.Certificate, error) {
certFile := c.toFilename(commonName, ".crt")
if storage.IsNotExist(c.store, certFile) {
glog.V(2).Infof("Issue %s certificate for %#v...", c.name, commonName)
c.mu.Lock()
defer c.mu.Unlock()
if storage.IsNotExist(c.store, certFile) {
if err := c.issue(commonName, vaildFor, rsaBits); err != nil {
return nil, err
}
}
}
tlsCert, err := tls.LoadX509KeyPair(certFile, certFile)
if err != nil {
return nil, err
}
return &tlsCert, nil
}
func (d *Dialer) Dial(network, address string) (conn net.Conn, err error) {
glog.V(3).Infof("Dail(%#v, %#v)", network, address)
switch network {
case "tcp", "tcp4", "tcp6":
if d.DNSCache != nil {
if addr, ok := d.DNSCache.Get(address); ok {
address = addr.(string)
} else {
if host, port, err := net.SplitHostPort(address); err == nil {
if ips, err := net.LookupIP(host); err == nil && len(ips) > 0 {
ip := ips[0].String()
if d.LoopbackAddrs != nil {
if _, ok := d.LoopbackAddrs[ip]; ok {
return nil, net.InvalidAddrError(fmt.Sprintf("Invaid DNS Record: %s(%s)", host, ip))
}
}
addr := net.JoinHostPort(ip, port)
expiry := d.DNSCacheExpiry
if expiry == 0 {
expiry = DefaultDNSCacheExpiry
}
d.DNSCache.Set(address, addr, time.Now().Add(expiry))
glog.V(3).Infof("direct Dial cache dns %#v=%#v", address, addr)
address = addr
}
}
}
}
default:
break
}
if d.Level <= 1 {
retry := d.RetryTimes
if retry == 0 {
retry = DefaultRetryTimes
}
for i := 0; i < retry; i++ {
conn, err = d.Dialer.Dial(network, address)
if err == nil || i == retry-1 {
break
}
retryDelay := d.RetryDelay
if retryDelay == 0 {
retryDelay = DefaultRetryDelay
}
time.Sleep(retryDelay)
}
return conn, err
} else {
type racer struct {
c net.Conn
e error
}
lane := make(chan racer, d.Level)
retry := (d.RetryTimes + d.Level - 1) / d.Level
for i := 0; i < retry; i++ {
for j := 0; j < d.Level; j++ {
go func(addr string, c chan<- racer) {
conn, err := d.Dialer.Dial(network, addr)
lane <- racer{conn, err}
}(address, lane)
}
var r racer
for k := 0; k < d.Level; k++ {
r = <-lane
if r.e == nil {
go func(count int) {
var r1 racer
for ; count > 0; count-- {
r1 = <-lane
if r1.c != nil {
r1.c.Close()
}
}
}(d.Level - 1 - k)
return r.c, nil
}
}
if i == retry-1 {
return nil, r.e
}
}
}
return nil, net.UnknownNetworkError("Unkown transport/direct error")
}
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
for i := 0; i < t.RetryTimes; i++ {
server := t.Servers.PickFetchServer(req, i)
req1, err := t.Servers.EncodeRequest(req, server)
if err != nil {
return nil, fmt.Errorf("GAE EncodeRequest: %s", err.Error())
}
resp, err := t.RoundTripper.RoundTrip(req1)
if err != nil {
isTimeoutError := false
if ne, ok := err.(interface {
Timeout() bool
}); ok && ne.Timeout() {
isTimeoutError = true
}
if ne, ok := err.(*net.OpError); ok && ne.Op == "read" {
isTimeoutError = true
}
if isTimeoutError {
glog.Warningf("GAE: \"%s %s\" timeout: %v, helpers.TryCloseConnections(%T)", req.Method, req.URL.String(), err, t.RoundTripper)
helpers.TryCloseConnections(t.RoundTripper)
}
if i == t.RetryTimes-1 {
return nil, err
} else {
glog.Warningf("GAE: request \"%s\" error: %T(%v), retry...", req.URL.String(), err, err)
continue
}
}
if resp.StatusCode != http.StatusOK {
if i == t.RetryTimes-1 {
return resp, nil
}
switch resp.StatusCode {
case http.StatusServiceUnavailable:
glog.Warningf("GAE: %s over qouta, try switch to next appid...", server.Host)
t.Servers.ToggleBadServer(server)
time.Sleep(t.RetryDelay)
continue
case http.StatusFound,
http.StatusBadGateway,
http.StatusNotFound,
http.StatusMethodNotAllowed:
if t.MultiDialer != nil {
if addr, err := helpers.ReflectRemoteAddrFromResponse(resp); err == nil {
if ip, _, err := net.SplitHostPort(addr); err == nil {
duration := 8 * time.Hour
glog.Warningf("GAE: %s StatusCode is %d, not a gws/gvs ip, add to blacklist for %v", ip, resp.StatusCode, duration)
t.MultiDialer.IPBlackList.Set(ip, struct{}{}, time.Now().Add(duration))
}
if ok := helpers.TryCloseConnectionByRemoteAddr(t.RoundTripper, addr); !ok {
glog.Warningf("GAE: TryCloseConnectionByRemoteAddr(%T, %#v) failed.", t.RoundTripper, addr)
}
}
}
continue
default:
return resp, nil
}
}
resp1, err := t.Servers.DecodeResponse(resp)
if err != nil {
return nil, err
}
if resp1 != nil {
resp1.Request = req
}
if i == t.RetryTimes-1 {
return resp, err
}
switch resp1.StatusCode {
case http.StatusBadGateway:
body, err := ioutil.ReadAll(resp1.Body)
if err != nil {
resp1.Body.Close()
return nil, err
}
resp1.Body.Close()
switch {
case bytes.Contains(body, []byte("DEADLINE_EXCEEDED")):
glog.V(2).Infof("GAE: %s urlfetch %#v get DEADLINE_EXCEEDED, retry...", req1.Host, req.URL.String())
continue
case bytes.Contains(body, []byte("ver quota")):
glog.V(2).Infof("GAE: %s urlfetch %#v get over quota, retry...", req1.Host, req.URL.String())
time.Sleep(t.RetryDelay)
continue
case bytes.Contains(body, []byte("urlfetch: CLOSED")):
glog.V(2).Infof("GAE: %s urlfetch %#v get urlfetch: CLOSED, retry...", req1.Host, req.URL.String())
time.Sleep(t.RetryDelay)
continue
default:
resp1.Body = ioutil.NopCloser(bytes.NewReader(body))
return resp1, nil
}
default:
return resp1, nil
}
}
return nil, fmt.Errorf("GAE: cannot reach here with %#v", req)
}
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
if !f.SiteMatcher.Match(req.Host) {
return ctx, nil, nil
}
var tr http.RoundTripper = f.GAETransport
if req.URL.Scheme == "http" && f.ForceHTTPSMatcher.Match(req.Host) {
if !strings.HasPrefix(req.Header.Get("Referer"), "https://") {
u := strings.Replace(req.URL.String(), "http://", "https://", 1)
glog.V(2).Infof("GAE FORCEHTTPS get raw url=%v, redirect to %v", req.URL.String(), u)
resp := &http.Response{
StatusCode: http.StatusMovedPermanently,
Header: http.Header{
"Location": []string{u},
},
Request: req,
Close: true,
ContentLength: -1,
}
glog.V(2).Infof("%s \"GAE FORCEHTTPS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
if f.DirectSiteMatcher.Match(req.Host) {
if req.URL.Path == "/url" {
if rawurl := req.URL.Query().Get("url"); rawurl != "" {
if u, err := url.Parse(rawurl); err == nil {
if u.Scheme == "http" && f.ForceHTTPSMatcher.Match(u.Host) {
rawurl = strings.Replace(rawurl, "http://", "https://", 1)
}
}
glog.V(2).Infof("%s \"GAE REDIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, rawurl, req.Proto)
return ctx, &http.Response{
StatusCode: http.StatusFound,
Header: http.Header{
"Location": []string{rawurl},
},
Request: req,
Close: true,
ContentLength: -1,
}, nil
}
}
if req.URL.Scheme != "http" && !f.shouldForceGAE(req) {
tr = f.DirectTransport
if s := req.Header.Get("Connection"); s != "" {
if s1 := strings.ToLower(s); s != s1 {
req.Header.Set("Connection", s1)
}
}
}
}
if tr != f.DirectTransport && req.Method == http.MethodOptions {
if v, ok := f.FakeOptionsMatcher.Lookup(req.Host); ok {
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{},
Request: req,
Close: false,
ContentLength: -1,
}
for _, s := range v.([]string) {
parts := strings.SplitN(s, ":", 2)
if len(parts) == 2 {
resp.Header.Add(parts[0], strings.TrimSpace(parts[1]))
}
}
if origin := req.Header.Get("Origin"); origin != "" {
resp.Header.Set("Access-Control-Allow-Origin", origin)
}
if headers := req.Header.Get("Access-Control-Request-Headers"); headers != "" {
resp.Header.Set("Access-Control-Allow-Headers", headers)
}
glog.V(2).Infof("%s \"GAE FAKEOPTIONS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
prefix := "FETCH"
if tr == f.DirectTransport {
prefix = "DIRECT"
}
resp, err := tr.RoundTrip(req)
if err != nil {
glog.Warningf("%s \"GAE %s %s %s %s\" error: %T(%v)", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, err, err)
if tr == f.DirectTransport {
if ne, ok := err.(interface {
Timeout() bool
}); ok && ne.Timeout() {
// f.MultiDialer.ClearCache()
helpers.TryCloseConnections(tr)
}
}
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
return ctx, nil, err
}
if strings.HasPrefix(resp.Header.Get("Content-Type"), "text/") &&
resp.Header.Get("Content-Encoding") == "" &&
f.ForceDeflateMatcher.Match(req.Host) {
buf := make([]byte, 1024)
n, err := resp.Body.Read(buf)
if err != nil {
defer resp.Body.Close()
return ctx, nil, err
}
buf = buf[:n]
switch {
case helpers.IsGzip(buf):
resp.Header.Set("Content-Encoding", "gzip")
case helpers.IsBinary(buf):
resp.Header.Set("Content-Encoding", "deflate")
}
resp.Body = helpers.NewMultiReadCloser(bytes.NewReader(buf), resp.Body)
}
if resp != nil && resp.Header != nil {
resp.Header.Del("Alt-Svc")
resp.Header.Del("Alternate-Protocol")
}
glog.V(2).Infof("%s \"GAE %s %s %s %s\" %d %s", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, err
}
func NewFilter(config *Config) (filters.Filter, error) {
dnsServers := make([]net.IP, 0)
for _, s := range config.DNSServers {
if ip := net.ParseIP(s); ip != nil {
dnsServers = append(dnsServers, ip)
}
}
GoogleG2PKP, err := base64.StdEncoding.DecodeString(config.GoogleG2PKP)
if err != nil {
return nil, err
}
googleTLSConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true,
ServerName: "www.microsoft.com",
ClientSessionCache: tls.NewLRUClientSessionCache(config.TLSConfig.ClientSessionCacheSize),
CipherSuites: []uint16{
tls.TLS_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
},
}
switch config.TLSConfig.Version {
case "TLSv12", "TLSv1.2":
googleTLSConfig.MinVersion = tls.VersionTLS12
case "TLSv11", "TLSv1.1":
googleTLSConfig.MinVersion = tls.VersionTLS11
default:
googleTLSConfig.MinVersion = tls.VersionTLS10
}
pickupCiphers := func(names []string) []uint16 {
ciphers := make([]uint16, 0)
for _, name := range names {
cipher := helpers.Cipher(name)
if cipher == 0 {
glog.Fatalf("GAE: cipher %#v is not supported.", name)
}
ciphers = append(ciphers, cipher)
}
helpers.ShuffleUint16s(ciphers)
ciphers = ciphers[:1+rand.Intn(len(ciphers))]
ciphers1 := []uint16{}
for _, name := range []string{
"TLS_RSA_WITH_AES_256_CBC_SHA256",
"TLS_RSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
} {
if !helpers.ContainsString(names, name) {
if c := helpers.Cipher(name); c != 0 {
ciphers1 = append(ciphers1, c)
}
}
}
helpers.ShuffleUint16s(ciphers1)
ciphers1 = ciphers1[:rand.Intn(len(ciphers1))]
ciphers = append(ciphers, ciphers1...)
helpers.ShuffleUint16s(ciphers)
return ciphers
}
googleTLSConfig.CipherSuites = pickupCiphers(config.TLSConfig.Ciphers)
if len(config.TLSConfig.ServerName) > 0 {
googleTLSConfig.ServerName = config.TLSConfig.ServerName[rand.Intn(len(config.TLSConfig.ServerName))]
}
if !config.DisableHTTP2 {
googleTLSConfig.NextProtos = []string{"h2", "h2-14", "http/1.1"}
}
if config.Site2Alias == nil {
config.Site2Alias = make(map[string]string)
}
for key, value := range config.SiteToAlias {
config.Site2Alias[key] = value
}
config.SiteToAlias = config.Site2Alias
hostmap := map[string][]string{}
for key, value := range config.HostMap {
hostmap[key] = helpers.UniqueStrings(value)
}
d := &net.Dialer{
KeepAlive: time.Duration(config.Transport.Dialer.KeepAlive) * time.Second,
Timeout: time.Duration(config.Transport.Dialer.Timeout) * time.Second,
DualStack: config.Transport.Dialer.DualStack,
}
md := &dialer.MultiDialer{
Dialer: d,
DisableIPv6: config.DisableIPv6,
ForceIPv6: config.ForceIPv6,
SSLVerify: config.SSLVerify,
EnableRemoteDNS: config.EnableRemoteDNS,
LogToStderr: flag.Lookup("logtostderr") != nil,
TLSConfig: nil,
SiteToAlias: helpers.NewHostMatcherWithString(config.SiteToAlias),
IPBlackList: lrucache.NewLRUCache(1024),
HostMap: hostmap,
GoogleTLSConfig: googleTLSConfig,
GoogleG2PKP: GoogleG2PKP,
DNSServers: dnsServers,
DNSCache: lrucache.NewLRUCache(config.Transport.Dialer.DNSCacheSize),
DNSCacheExpiry: time.Duration(config.Transport.Dialer.DNSCacheExpiry) * time.Second,
TLSConnDuration: lrucache.NewLRUCache(8192),
TLSConnError: lrucache.NewLRUCache(8192),
TLSConnReadBuffer: config.Transport.Dialer.SocketReadBuffer,
ConnExpiry: 5 * time.Minute,
Level: config.Transport.Dialer.Level,
}
for _, ip := range config.IPBlackList {
md.IPBlackList.Set(ip, struct{}{}, time.Time{})
}
var tr http.RoundTripper
t1 := &http.Transport{
Dial: md.Dial,
DialTLS: md.DialTLS,
DisableKeepAlives: config.Transport.DisableKeepAlives,
DisableCompression: config.Transport.DisableCompression,
ResponseHeaderTimeout: time.Duration(config.Transport.ResponseHeaderTimeout) * time.Second,
IdleConnTimeout: time.Duration(config.Transport.IdleConnTimeout) * time.Second,
MaxIdleConnsPerHost: config.Transport.MaxIdleConnsPerHost,
}
if config.Transport.Proxy.Enabled {
fixedURL, err := url.Parse(config.Transport.Proxy.URL)
if err != nil {
glog.Fatalf("url.Parse(%#v) error: %s", config.Transport.Proxy.URL, err)
}
dialer, err := proxy.FromURL(fixedURL, d, &dialer.MultiResolver{md})
if err != nil {
glog.Fatalf("proxy.FromURL(%#v) error: %s", fixedURL.String(), err)
}
t1.Dial = dialer.Dial
t1.DialTLS = nil
t1.Proxy = nil
t1.TLSClientConfig = md.GoogleTLSConfig
}
switch {
case config.DisableHTTP2 && config.ForceHTTP2:
glog.Fatalf("GAE: DisableHTTP2=%v and ForceHTTP2=%v is conflict!", config.DisableHTTP2, config.ForceHTTP2)
case config.Transport.Proxy.Enabled && config.ForceHTTP2:
glog.Fatalf("GAE: Proxy.Enabled=%v and ForceHTTP2=%v is conflict!", config.Transport.Proxy.Enabled, config.ForceHTTP2)
case config.ForceHTTP2:
tr = &http2.Transport{
DialTLS: md.DialTLS2,
TLSClientConfig: md.GoogleTLSConfig,
DisableCompression: config.Transport.DisableCompression,
}
case !config.DisableHTTP2:
err := http2.ConfigureTransport(t1)
if err != nil {
glog.Warningf("GAE: Error enabling Transport HTTP/2 support: %v", err)
}
tr = t1
default:
tr = t1
}
forceHTTPSMatcherStrings := make([]string, 0)
for key, value := range config.SiteToAlias {
if strings.HasPrefix(value, "google_") {
forceHTTPSMatcherStrings = append(forceHTTPSMatcherStrings, key)
}
}
forceGAEStrings := make([]string, 0)
forceGAESuffixs := make([]string, 0)
forceGAEMatcherStrings := make([]string, 0)
for _, s := range config.ForceGAE {
if strings.Contains(s, "/") {
if strings.HasSuffix(s, "$") {
forceGAESuffixs = append(forceGAESuffixs, strings.TrimRight(s, "$"))
} else {
forceGAEStrings = append(forceGAEStrings, s)
}
} else {
forceGAEMatcherStrings = append(forceGAEMatcherStrings, s)
}
}
if config.EnableDeadProbe && !config.Transport.Proxy.Enabled {
go func() {
probe := func() {
req, _ := http.NewRequest(http.MethodGet, "https://clients3.google.com/generate_204", nil)
ctx, cancel := context.WithTimeout(req.Context(), 3*time.Second)
defer cancel()
req = req.WithContext(ctx)
resp, err := tr.RoundTrip(req)
if resp != nil && resp.Body != nil {
glog.V(3).Infof("GAE EnableDeadProbe \"%s %s\" %d -", req.Method, req.URL.String(), resp.StatusCode)
resp.Body.Close()
}
if err != nil {
glog.V(2).Infof("GAE EnableDeadProbe \"%s %s\" error: %v", req.Method, req.URL.String(), err)
s := strings.ToLower(err.Error())
if strings.HasPrefix(s, "net/http: request canceled") || strings.Contains(s, "timeout") {
helpers.TryCloseConnections(tr)
}
}
}
for {
time.Sleep(time.Duration(5+rand.Intn(6)) * time.Second)
probe()
}
}()
}
helpers.ShuffleStrings(config.AppIDs)
f := &Filter{
Config: *config,
GAETransport: &Transport{
RoundTripper: tr,
MultiDialer: md,
Servers: NewServers(config.AppIDs,
config.Password,
config.SSLVerify,
time.Duration(config.Transport.ResponseHeaderTimeout-4)*time.Second),
RetryDelay: time.Duration(config.Transport.RetryDelay*1000) * time.Second,
RetryTimes: config.Transport.RetryTimes,
},
DirectTransport: tr,
ForceHTTPSMatcher: helpers.NewHostMatcher(forceHTTPSMatcherStrings),
ForceGAEMatcher: helpers.NewHostMatcher(forceGAEMatcherStrings),
ForceGAEStrings: forceGAEStrings,
ForceGAESuffixs: forceGAESuffixs,
ForceDeflateMatcher: helpers.NewHostMatcher(config.ForceDeflate),
FakeOptionsMatcher: helpers.NewHostMatcherWithStrings(config.FakeOptions),
SiteMatcher: helpers.NewHostMatcher(config.Sites),
DirectSiteMatcher: helpers.NewHostMatcherWithString(config.Site2Alias),
}
if config.Transport.Proxy.Enabled {
f.GAETransport.MultiDialer = nil
}
return f, nil
}
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
var tr http.RoundTripper = f.GAETransport
if req.URL.Scheme == "http" && f.ForceHTTPSMatcher.Match(req.Host) {
if !strings.HasPrefix(req.Header.Get("Referer"), "https://") {
u := strings.Replace(req.URL.String(), "http://", "https://", 1)
glog.V(2).Infof("GAE FORCEHTTPS get raw url=%v, redirect to %v", req.URL.String(), u)
resp := &http.Response{
StatusCode: http.StatusMovedPermanently,
Header: http.Header{
"Location": []string{u},
},
Request: req,
Close: true,
ContentLength: -1,
}
glog.V(2).Infof("%s \"GAE FORCEHTTPS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
if f.DirectSiteMatcher.Match(req.Host) {
switch req.URL.Path {
case "/url":
if rawurl := req.URL.Query().Get("url"); rawurl != "" {
if u, err := url.Parse(rawurl); err == nil {
if u.Scheme == "http" && f.ForceHTTPSMatcher.Match(u.Host) {
rawurl = strings.Replace(rawurl, "http://", "https://", 1)
}
}
glog.V(2).Infof("%s \"GAE REDIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, rawurl, req.Proto)
return ctx, &http.Response{
StatusCode: http.StatusFound,
Header: http.Header{
"Location": []string{rawurl},
},
Request: req,
Close: true,
ContentLength: -1,
}, nil
}
case "/books":
if req.URL.Host == "books.google.cn" {
rawurl := strings.Replace(req.URL.String(), "books.google.cn", "books.google.com", 1)
glog.V(2).Infof("%s \"GAE REDIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, rawurl, req.Proto)
return ctx, &http.Response{
StatusCode: http.StatusFound,
Header: http.Header{
"Location": []string{rawurl},
},
Request: req,
Close: true,
ContentLength: -1,
}, nil
}
}
if req.URL.Scheme != "http" && !f.shouldForceGAE(req) {
tr = f.DirectTransport
if s := req.Header.Get("Connection"); s != "" {
if s1 := strings.ToLower(s); s != s1 {
req.Header.Set("Connection", s1)
}
}
}
}
if tr != f.DirectTransport && req.Method == http.MethodOptions {
if v, ok := f.FakeOptionsMatcher.Lookup(req.Host); ok {
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{},
Request: req,
Close: false,
ContentLength: -1,
}
for _, s := range v.([]string) {
parts := strings.SplitN(s, ":", 2)
if len(parts) == 2 {
resp.Header.Add(parts[0], strings.TrimSpace(parts[1]))
}
}
if origin := req.Header.Get("Origin"); origin != "" {
resp.Header.Set("Access-Control-Allow-Origin", origin)
}
if headers := req.Header.Get("Access-Control-Request-Headers"); headers != "" {
resp.Header.Set("Access-Control-Allow-Headers", headers)
}
glog.V(2).Infof("%s \"GAE FAKEOPTIONS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
if tr == f.GAETransport {
switch strings.ToLower(req.Header.Get("Connection")) {
case "upgrade":
resp := &http.Response{
StatusCode: http.StatusForbidden,
Header: http.Header{
"X-WebSocket-Reject-Reason": []string{"Unsupported"},
},
Request: req,
Close: true,
ContentLength: 0,
}
glog.V(2).Infof("%s \"GAE FAKEWEBSOCKET %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
prefix := "FETCH"
if tr == f.DirectTransport {
prefix = "DIRECT"
}
resp, err := tr.RoundTrip(req)
if err != nil {
glog.Warningf("%s \"GAE %s %s %s %s\" error: %T(%v)", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, err, err)
if tr == f.DirectTransport {
if ne, ok := err.(interface {
Timeout() bool
}); ok && ne.Timeout() {
// f.MultiDialer.ClearCache()
helpers.CloseConnections(tr)
}
}
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
return ctx, nil, err
}
if tr == f.DirectTransport && resp.StatusCode >= http.StatusBadRequest {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
resp.Body.Close()
return ctx, nil, err
}
if addr, err := helpers.ReflectRemoteAddrFromResponse(resp); err == nil {
if ip, _, err := net.SplitHostPort(addr); err == nil {
var duration time.Duration
switch {
case resp.StatusCode == http.StatusBadGateway && bytes.Contains(body, []byte("Please try again in 30 seconds.")):
duration = 1 * time.Hour
case resp.StatusCode == http.StatusNotFound && bytes.Contains(body, []byte("<ins>That’s all we know.</ins>")):
server := resp.Header.Get("Server")
if server != "gws" && !strings.HasPrefix(server, "gvs") {
if md := f.GAETransport.MultiDialer; md != nil && md.TLSConnDuration.Len() > 10 {
duration = 5 * time.Minute
}
}
}
if duration > 0 && f.GAETransport.MultiDialer != nil {
glog.Warningf("GAE: %s StatusCode is %d, not a gws/gvs ip, add to blacklist for %v", ip, resp.StatusCode, duration)
f.GAETransport.MultiDialer.IPBlackList.Set(ip, struct{}{}, time.Now().Add(duration))
if !helpers.CloseConnectionByRemoteAddr(tr, addr) {
glog.Warningf("GAE: CloseConnectionByRemoteAddr(%T, %#v) failed.", tr, addr)
}
}
}
}
resp.Body.Close()
resp.Body = ioutil.NopCloser(bytes.NewReader(body))
}
if resp != nil && resp.Header != nil {
resp.Header.Del("Alt-Svc")
resp.Header.Del("Alternate-Protocol")
}
glog.V(2).Infof("%s \"GAE %s %s %s %s\" %d %s", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, err
}
func (d *MultiDialer) DialTLS2(network, address string, cfg *tls.Config) (net.Conn, error) {
if d.LogToStderr {
SetConsoleTextColorGreen()
}
glog.V(2).Infof("MULTIDIALER DialTLS2(%#v, %#v) with good_addrs=%d, bad_addrs=%d", network, address, d.TLSConnDuration.Len(), d.TLSConnError.Len())
if d.LogToStderr {
SetConsoleTextColorReset()
}
if cfg == nil {
cfg = d.TLSConfig
}
switch network {
case "tcp", "tcp4", "tcp6":
if host, port, err := net.SplitHostPort(address); err == nil {
if alias0, ok := d.SiteToAlias.Lookup(host); ok {
alias := alias0.(string)
if hosts, err := d.LookupAlias(alias); err == nil {
var config *tls.Config
isGoogleAddr := false
switch {
case strings.HasPrefix(alias, "google_"):
config = d.GoogleTLSConfig
isGoogleAddr = true
case cfg == nil:
config = &tls.Config{
InsecureSkipVerify: !d.SSLVerify,
ServerName: address,
}
default:
config = cfg
}
glog.V(3).Infof("DialTLS2(%#v, %#v) alais=%#v set tls.Config=%#v", network, address, alias, config)
addrs := make([]string, len(hosts))
for i, host := range hosts {
addrs[i] = net.JoinHostPort(host, port)
}
switch {
case d.Resolver.ForceIPv6:
network = "tcp6"
case d.Resolver.DisableIPv6:
network = "tcp4"
}
conn, err := d.dialMultiTLS(network, addrs, config)
if err != nil {
return nil, err
}
if d.SSLVerify && isGoogleAddr {
if tc, ok := conn.(*tls.Conn); ok {
certs := tc.ConnectionState().PeerCertificates
if len(tc.ConnectionState().PeerCertificates) <= 1 {
return nil, fmt.Errorf("Wrong certificate of %s: PeerCertificates=%#v", conn.RemoteAddr(), certs)
}
cert := certs[1]
glog.V(3).Infof("MULTIDIALER DialTLS(%#v, %#v) verify cert=%v", network, address, cert.Subject)
if d.GoogleG2PKP != nil {
pkp := sha256.Sum256(cert.RawSubjectPublicKeyInfo)
if !bytes.Equal(pkp[:], d.GoogleG2PKP) {
defer conn.Close()
return nil, fmt.Errorf("Wrong certificate of %s: Issuer=%v, SubjectKeyId=%#v", conn.RemoteAddr(), cert.Subject, cert.SubjectKeyId)
}
} else {
if !strings.HasPrefix(cert.Subject.CommonName, "Google ") {
defer conn.Close()
return nil, fmt.Errorf("Wrong certificate of %s: Issuer=%v, SubjectKeyId=%#v", conn.RemoteAddr(), cert.Subject, cert.SubjectKeyId)
}
}
}
}
return conn, nil
}
}
}
default:
break
}
if dialer, ok := d.Dialer.(*net.Dialer); ok {
return tls.DialWithDialer(dialer, network, address, d.TLSConfig)
} else {
return tls.Dial(network, address, d.TLSConfig)
}
}
func (f *Filter) ProxyPacRoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
filename := req.URL.Path[1:]
data := ""
obj, err := f.Store.GetObject(filename, -1, -1)
switch {
case os.IsNotExist(err):
glog.V(2).Infof("AUTOPROXY ProxyPac: generate %#v", filename)
s := fmt.Sprintf(`// User-defined FindProxyForURL
function FindProxyForURL(url, host) {
if (shExpMatch(host, '*.google*.*') ||
dnsDomainIs(host, '.ggpht.com') ||
dnsDomainIs(host, '.gstatic.com') ||
host == 'goo.gl') {
return 'PROXY %s';
}
return 'DIRECT';
}
`, req.Host)
f.Store.PutObject(filename, http.Header{}, ioutil.NopCloser(bytes.NewBufferString(s)))
case err != nil:
return ctx, nil, err
case obj != nil:
if body := obj.Body(); body != nil {
body.Close()
}
}
if obj, err := f.Store.GetObject(filename, -1, -1); err == nil {
body := obj.Body()
defer body.Close()
if b, err := ioutil.ReadAll(body); err == nil {
s := strings.Replace(string(b), "function FindProxyForURL(", "function MyFindProxyForURL(", 1)
host, _, err := net.SplitHostPort(req.Host)
if err != nil {
host = req.Host
}
for _, localaddr := range []string{"127.0.0.1:", "[::1]:", "localhost:"} {
s = strings.Replace(s, localaddr, net.JoinHostPort(host, ""), -1)
}
data += s
}
}
if f.GFWListEnabled {
data += f.AutoProxy2Pac.GeneratePac(req)
} else {
data += `
function FindProxyForURL(url, host) {
if (isPlainHostName(host) ||
host.indexOf('127.') == 0 ||
host.indexOf('192.168.') == 0 ||
host.indexOf('10.') == 0 ||
shExpMatch(host, 'localhost.*')) {
return 'DIRECT';
}
return MyFindProxyForURL(url, host);
}`
}
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{},
Request: req,
Close: true,
ContentLength: int64(len(data)),
Body: ioutil.NopCloser(bytes.NewReader([]byte(data))),
}
return ctx, resp, nil
}
func (f *Filter) ProxyPacRoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
_, port, err := net.SplitHostPort(req.Host)
if err != nil {
port = "80"
}
if v, ok := f.ProxyPacCache.Get(req.RequestURI); ok {
if s, ok := v.(string); ok {
s = fixProxyPac(s, req)
return ctx, &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{},
Request: req,
Close: true,
ContentLength: int64(len(s)),
Body: ioutil.NopCloser(strings.NewReader(s)),
}, nil
}
}
filename := req.URL.Path[1:]
buf := new(bytes.Buffer)
resp, err := f.Store.Get(filename, -1, -1)
switch {
case os.IsNotExist(err), resp.StatusCode == http.StatusNotFound:
glog.V(2).Infof("AUTOPROXY ProxyPac: generate %#v", filename)
s := fmt.Sprintf(`// User-defined FindProxyForURL
function FindProxyForURL(url, host) {
if (isPlainHostName(host) ||
host.indexOf('127.') == 0 ||
host.indexOf('192.168.') == 0 ||
host.indexOf('10.') == 0 ||
shExpMatch(host, 'localhost.*')) {
return 'DIRECT';
}
if (shExpMatch(host, '*.google*.*')) {
return 'PROXY localhost:%s';
}
return 'DIRECT';
}
`, port)
f.Store.Put(filename, http.Header{}, ioutil.NopCloser(bytes.NewBufferString(s)))
case err != nil:
return ctx, nil, err
case resp.Body != nil:
resp.Body.Close()
}
if resp, err := f.Store.Get(filename, -1, -1); err == nil {
defer resp.Body.Close()
if b, err := ioutil.ReadAll(resp.Body); err == nil {
if f.GFWListEnabled {
b = []byte(strings.Replace(string(b), "function FindProxyForURL(", "function MyFindProxyForURL(", 1))
}
buf.Write(b)
}
}
if f.GFWListEnabled {
resp, err := f.Store.Get(f.GFWList.Filename, -1, -1)
if err != nil {
glog.Errorf("GetObject(%#v) error: %v", f.GFWList.Filename, err)
return ctx, nil, err
}
defer resp.Body.Close()
sites, err := parseAutoProxy(resp.Body)
if err != nil {
glog.Errorf("parseAutoProxy(%#v) error: %v", f.GFWList.Filename, err)
return ctx, nil, err
}
sort.Strings(sites)
io.WriteString(buf, "\nvar sites = {\n")
for _, site := range sites {
io.WriteString(buf, "\""+site+"\":1,\n")
}
io.WriteString(buf, "\"google.com\":1\n")
io.WriteString(buf, "}\n")
io.WriteString(buf, `
function FindProxyForURL(url, host) {
if ((p = MyFindProxyForURL(url, host)) != "DIRECT") {
return p
}
var lastPos;
do {
if (sites.hasOwnProperty(host)) {
return 'PROXY GOPROXY_ADDRESS';
}
lastPos = host.indexOf('.') + 1;
host = host.slice(lastPos);
} while (lastPos >= 1);
return 'DIRECT';
}`)
}
s := buf.String()
f.ProxyPacCache.Set(req.RequestURI, s, time.Now().Add(15*time.Minute))
s = fixProxyPac(s, req)
resp = &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{},
Request: req,
Close: true,
ContentLength: int64(len(s)),
Body: ioutil.NopCloser(strings.NewReader(s)),
}
return ctx, resp, nil
}
func (f *Filter) updater() {
glog.V(2).Infof("start updater for %+v, expiry=%s, duration=%s", f.GFWList.URL.String(), f.GFWList.Expiry, f.GFWList.Duration)
ticker := time.Tick(f.GFWList.Duration)
for {
select {
case <-ticker:
glog.V(2).Infof("Begin auto gfwlist(%#v) update...", f.GFWList.URL.String())
h, err := f.Store.HeadObject(f.GFWList.Filename)
if err != nil {
glog.Warningf("stat gfwlist(%#v) err: %v", f.GFWList.Filename, err)
continue
}
lm := h.Get("Last-Modified")
if lm == "" {
glog.Warningf("gfwlist(%#v) header(%#v) does not contains last-modified", f.GFWList.Filename, h)
continue
}
modTime, err := time.Parse(f.Store.DateFormat(), lm)
if err != nil {
glog.Warningf("stat gfwlist(%#v) has parse %#v error: %v", f.GFWList.Filename, lm, err)
continue
}
if time.Now().Sub(modTime) < f.GFWList.Expiry {
continue
}
}
glog.Infof("Downloading %#v", f.GFWList.URL.String())
req, err := http.NewRequest(http.MethodGet, f.GFWList.URL.String(), nil)
if err != nil {
glog.Warningf("NewRequest(%#v) error: %v", f.GFWList.URL.String(), err)
continue
}
resp, err := f.Transport.RoundTrip(req)
if err != nil {
glog.Warningf("%T.RoundTrip(%#v) error: %v", f.Transport, f.GFWList.URL.String(), err.Error())
continue
}
var r io.Reader = resp.Body
switch f.GFWList.Encoding {
case "base64":
r = base64.NewDecoder(base64.StdEncoding, r)
default:
break
}
data, err := ioutil.ReadAll(r)
if err != nil {
glog.Warningf("ioutil.ReadAll(%T) error: %v", r, err)
resp.Body.Close()
continue
}
err = f.Store.DeleteObject(f.GFWList.Filename)
if err != nil {
glog.Warningf("%T.DeleteObject(%#v) error: %v", f.Store, f.GFWList.Filename, err)
continue
}
err = f.Store.PutObject(f.GFWList.Filename, http.Header{}, ioutil.NopCloser(bytes.NewReader(data)))
if err != nil {
glog.Warningf("%T.PutObject(%#v) error: %v", f.Store, f.GFWList.Filename, err)
continue
}
glog.Infof("Update %#v from %#v OK", f.GFWList.Filename, f.GFWList.URL.String())
resp.Body.Close()
}
}
func (f *Filter) Response(ctx context.Context, resp *http.Response) (context.Context, *http.Response, error) {
if resp.StatusCode != http.StatusPartialContent || resp.Header.Get("Content-Length") == "" {
return ctx, resp, nil
}
if ok1, ok := filters.Bool(ctx, "autorange.default"); ok && ok1 {
return ctx, resp, nil
}
f1 := filters.GetRoundTripFilter(ctx)
if f1 == nil {
return ctx, resp, nil
}
if _, ok := f.SupportFilters[f1.FilterName()]; !ok {
glog.V(2).Infof("AUTORANGE hit a unsupported filter=%#v", f1)
return ctx, resp, nil
}
parts := strings.Split(resp.Header.Get("Content-Range"), " ")
if len(parts) != 2 || parts[0] != "bytes" {
return ctx, resp, nil
}
parts1 := strings.Split(parts[1], "/")
parts2 := strings.Split(parts1[0], "-")
if len(parts1) != 2 || len(parts2) != 2 {
return ctx, resp, nil
}
var end, length int64
var err error
end, err = strconv.ParseInt(parts2[1], 10, 64)
if err != nil {
return ctx, resp, nil
}
length, err = strconv.ParseInt(parts1[1], 10, 64)
if err != nil {
return ctx, resp, nil
}
glog.V(2).Infof("AUTORANGE respone matched, start rangefetch for %#v", resp.Header.Get("Content-Range"))
resp.ContentLength = length
resp.Header.Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10))
resp.Header.Del("Content-Range")
r, w := AutoPipe(f.Threads)
go func(w *autoPipeWriter, filter filters.RoundTripFilter, req0 *http.Request, start, length int64) {
glog.V(2).Infof("AUTORANGE begin rangefetch for %#v by using %#v", req0.URL.String(), filter.FilterName())
req, err := http.NewRequest(req0.Method, req0.URL.String(), nil)
if err != nil {
glog.Warningf("AUTORANGE http.NewRequest(%#v) error: %#v", req, err)
return
}
for key, values := range req0.Header {
for _, value := range values {
req.Header.Add(key, value)
}
}
if err := w.WaitForReading(); err != nil {
return
}
var index uint32
for {
if w.FatalErr() {
break
}
if start > length-1 {
// tell reader io.EOF
w.Close()
break
}
//FIXME: make this configurable!
if w.Len() > 128*1024*1024 {
time.Sleep(100 * time.Millisecond)
continue
}
//end := start + int64(f.BufSize-1)
end := start + int64(1024<<10-1)
if end > length-1 {
end = length - 1
}
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
_, resp, err := filter.RoundTrip(nil, req)
if err != nil {
glog.Warningf("AUTORANGE %#v.RoundTrip(%v) error: %#v", filter, req, err)
time.Sleep(1 * time.Second)
continue
}
if resp.StatusCode != http.StatusPartialContent {
if resp.StatusCode >= http.StatusBadRequest {
time.Sleep(1 * time.Second)
}
continue
}
w.ThreadHello()
go func(index uint32, resp *http.Response) {
defer resp.Body.Close()
defer w.ThreadBye() // TODO: launch it as soon as iocopy over?
piper := w.NewPiper(index)
_, err := helpers.IoCopy(piper, resp.Body)
if err != nil {
glog.Warningf("AUTORANGE helpers.IoCopy(%#v) error: %#v", resp.Body, err)
piper.EIndex()
}
piper.WClose()
}(index, resp)
start = end + 1
index++
}
}(w, f1, resp.Request, end+1, length)
resp.Body = helpers.NewMultiReadCloser(resp.Body, r)
return ctx, resp, nil
}
