コード例 #1
0
ファイル: autorange.go プロジェクト: taiping-z/goproxy
func (f *Filter) Request(ctx context.Context, req *http.Request) (context.Context, *http.Request, error) {
	if req.Method != http.MethodGet || strings.Contains(req.URL.RawQuery, "range=") {
		return ctx, req, nil
	}

	if r := req.Header.Get("Range"); r == "" {
		switch {
		case f.SiteMatcher.Match(req.Host):
			req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", 0, f.MaxSize))
			glog.V(2).Infof("AUTORANGE Sites rule matched, add %s for\"%s\"", req.Header.Get("Range"), req.URL.String())
			ctx = filters.WithBool(ctx, "autorange.site", true)
		default:
			glog.V(3).Infof("AUTORANGE ignore preserved empty range for %#v", req.URL)
		}
	} else {
		ctx = filters.WithBool(ctx, "autorange.default", true)
		parts := strings.Split(r, "=")
		switch parts[0] {
		case "bytes":
			parts1 := strings.Split(parts[1], "-")
			if start, err := strconv.Atoi(parts1[0]); err == nil {
				if end, err := strconv.Atoi(parts1[1]); err != nil || end-start > f.MaxSize {
					req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, start+f.MaxSize))
					glog.V(2).Infof("AUTORANGE Default rule matched, change %s to %s for\"%s\"", r, req.Header.Get("Range"), req.URL.String())
				}
			}
		default:
			glog.Warningf("AUTORANGE Default rule matched, but cannot support %#v range for \"%s\"", r, req.URL.String())
		}
	}

	return ctx, req, nil
}
コード例 #2
0
ファイル: autoproxy.go プロジェクト: chenzhenjia/goproxy
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
	if f.SiteFiltersEnabled {
		if f1, ok := f.SiteFiltersRules.Lookup(req.Host); ok {
			glog.V(2).Infof("%s \"AUTOPROXY SiteFilters %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
			return f1.(filters.RoundTripFilter).RoundTrip(ctx, req)
		}
	}

	if f.RegionFiltersEnabled {
		//TODO
	}

	if req.URL.Host == "" && req.RequestURI[0] == '/' && f.IndexFilesEnabled {
		if _, ok := f.IndexFiles[req.URL.Path[1:]]; ok || req.URL.Path == "/" {
			if f.GFWListEnabled && strings.HasSuffix(req.URL.Path, ".pac") {
				glog.V(2).Infof("%s \"AUTOPROXY ProxyPac %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
				return f.ProxyPacRoundTrip(ctx, req)
			} else {
				glog.V(2).Infof("%s \"AUTOPROXY IndexFiles %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
				return f.IndexFilesRoundTrip(ctx, req)
			}
		}
	}

	return ctx, nil, nil
}
コード例 #3
0
ファイル: direct.go プロジェクト: chenzhenjia/goproxy
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
	switch req.Method {
	case "CONNECT":
		glog.V(2).Infof("%s \"DIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, req.Host, req.Proto)
		rconn, err := f.transport.Dial("tcp", req.Host)
		if err != nil {
			return ctx, nil, err
		}

		rw := filters.GetResponseWriter(ctx)

		hijacker, ok := rw.(http.Hijacker)
		if !ok {
			return ctx, nil, fmt.Errorf("http.ResponseWriter(%#v) does not implments http.Hijacker", rw)
		}

		flusher, ok := rw.(http.Flusher)
		if !ok {
			return ctx, nil, fmt.Errorf("http.ResponseWriter(%#v) does not implments http.Flusher", rw)
		}

		rw.WriteHeader(http.StatusOK)
		flusher.Flush()

		lconn, _, err := hijacker.Hijack()
		if err != nil {
			return ctx, nil, fmt.Errorf("%#v.Hijack() error: %v", hijacker, err)
		}
		defer lconn.Close()

		go helpers.IoCopy(rconn, lconn)
		helpers.IoCopy(lconn, rconn)

		filters.SetHijacked(ctx, true)
		return ctx, nil, nil
	default:
		helpers.FixRequestURL(req)
		resp, err := f.transport.RoundTrip(req)

		if err != nil {
			glog.Errorf("%s \"DIRECT %s %s %s\" error: %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, err)
			data := err.Error()
			resp = &http.Response{
				StatusCode:    http.StatusBadGateway,
				Header:        http.Header{},
				Request:       req,
				Close:         true,
				ContentLength: int64(len(data)),
				Body:          ioutil.NopCloser(bytes.NewReader([]byte(data))),
			}
			err = nil
		} else {
			if req.RemoteAddr != "" {
				glog.V(2).Infof("%s \"DIRECT %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
			}
		}
		return ctx, resp, err
	}
}
コード例 #4
0
ファイル: auth.go プロジェクト: taiping-z/goproxy
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {

	if ip, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
		if _, ok := f.WhiteList[ip]; ok {
			return ctx, nil, nil
		}
	}

	if auth := filters.String(ctx, authHeader); auth != "" {
		if _, ok := f.ByPassHeaders.Get(auth); ok {
			glog.V(3).Infof("auth filter hit bypass cache %#v", auth)
			return ctx, nil, nil
		}
		parts := strings.SplitN(auth, " ", 2)
		if len(parts) == 2 {
			switch parts[0] {
			case "Basic":
				if userpass, err := base64.StdEncoding.DecodeString(parts[1]); err == nil {
					parts := strings.Split(string(userpass), ":")
					user := parts[0]
					pass := parts[1]
					pass1, ok := f.Basic[user]
					if ok && pass == pass1 {
						f.ByPassHeaders.Set(auth, struct{}{}, time.Now().Add(time.Hour))
						return ctx, nil, nil
					}
				}
			default:
				glog.Errorf("Unrecognized auth type: %#v", parts[0])
				break
			}
		}
	}

	glog.V(1).Infof("UnAuthenticated URL %v from %#v", req.URL.String(), req.RemoteAddr)

	noAuthResponse := &http.Response{
		Status:        "407 Proxy Authentication Required",
		StatusCode:    http.StatusProxyAuthRequired,
		Proto:         "HTTP/1.1",
		ProtoMajor:    1,
		ProtoMinor:    1,
		Header:        http.Header{},
		Request:       req,
		Close:         true,
		ContentLength: -1,
	}

	return ctx, noAuthResponse, nil
}
コード例 #5
0
ファイル: direct.go プロジェクト: johnsonz/goproxy
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
	switch req.Method {
	case "CONNECT":
		glog.V(2).Infof("%s \"DIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, req.Host, req.Proto)
		rconn, err := f.transport.Dial("tcp", req.Host)
		if err != nil {
			return ctx, nil, err
		}

		rw := filters.GetResponseWriter(ctx)

		hijacker, ok := rw.(http.Hijacker)
		if !ok {
			return ctx, nil, fmt.Errorf("http.ResponseWriter(%#v) does not implments http.Hijacker", rw)
		}

		flusher, ok := rw.(http.Flusher)
		if !ok {
			return ctx, nil, fmt.Errorf("http.ResponseWriter(%#v) does not implments http.Flusher", rw)
		}

		rw.WriteHeader(http.StatusOK)
		flusher.Flush()

		lconn, _, err := hijacker.Hijack()
		if err != nil {
			return ctx, nil, fmt.Errorf("%#v.Hijack() error: %v", hijacker, err)
		}
		defer lconn.Close()

		go helpers.IOCopy(rconn, lconn)
		helpers.IOCopy(lconn, rconn)

		return ctx, filters.DummyResponse, nil
	default:
		helpers.FixRequestURL(req)
		helpers.FixRequestHeader(req)
		resp, err := f.transport.RoundTrip(req)

		if err != nil {
			return ctx, nil, err
		}

		if req.RemoteAddr != "" {
			glog.V(2).Infof("%s \"DIRECT %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
		}

		return ctx, resp, err
	}
}
コード例 #6
0
ファイル: rootca.go プロジェクト: phuslu/goagent
func (c *RootCA) Issue(commonName string, vaildFor time.Duration, rsaBits int) (*tls.Certificate, error) {
	certFile := c.toFilename(commonName, ".crt")

	if storage.NotExist(c.store, certFile) {
		glog.V(2).Infof("Issue %s certificate for %#v...", c.name, commonName)
		c.mu.Lock()
		defer c.mu.Unlock()
		if storage.NotExist(c.store, certFile) {
			if err := c.issue(commonName, vaildFor, rsaBits); err != nil {
				return nil, err
			}
		}
	}

	resp, err := c.store.Get(certFile)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	tlsCert, err := tls.X509KeyPair(data, data)
	if err != nil {
		return nil, err
	}
	return &tlsCert, nil
}
コード例 #7
0
ファイル: rewrite.go プロジェクト: johnsonz/goproxy
func (f *Filter) Request(ctx context.Context, req *http.Request) (context.Context, *http.Request, error) {
	if f.UserAgentEnabled {
		glog.V(3).Infof("REWRITE %#v User-Agent=%#v", req.URL.String(), f.UserAgentValue)
		req.Header.Set("User-Agent", f.UserAgentValue)
	}

	if f.HostEnabled {
		if host := req.Header.Get(f.HostRewriteBy); host != "" {
			glog.V(3).Infof("REWRITE %#v Host=%#v", req.URL.String(), host)
			req.Host = host
			req.Header.Set("Host", req.Host)
			req.Header.Del(f.HostRewriteBy)
		}
	}

	return ctx, req, nil
}
コード例 #8
0
ファイル: rewrite.go プロジェクト: ChoyesYan/goproxy
func (f *Filter) Request(ctx context.Context, req *http.Request) (context.Context, *http.Request, error) {
	if f.UserAgentEnabled {
		glog.V(3).Infof("REWRITE %#v User-Agent=%#v", req.URL.String(), f.UserAgentValue)
		req.Header.Set("User-Agent", f.UserAgentValue)
	}

	return ctx, req, nil
}
コード例 #9
0
ファイル: multidialer.go プロジェクト: ChoyesYan/goproxy
func (d *MultiDialer) ExpandAlias(alias string) error {
	names, ok := d.HostMap[alias]
	if !ok {
		return fmt.Errorf("alias %#v not exists", alias)
	}

	expire := time.Now().Add(24 * time.Hour)
	for _, name := range names {
		seen := make(map[string]struct{}, 0)
		for _, dnsserver := range d.DNSServers {
			var addrs []string
			var err error
			if net.ParseIP(name) != nil {
				addrs = []string{name}
				expire = time.Time{}
			} else if addrs, err = d.lookupHost2(name, dnsserver); err != nil {
				glog.V(2).Infof("lookupHost2(%#v) error: %s", name, err)
				continue
			}
			glog.V(2).Infof("ExpandList(%#v) %#v return %v", name, dnsserver, addrs)
			for _, addr := range addrs {
				seen[addr] = struct{}{}
			}
		}

		if len(seen) == 0 {
			continue
		}

		if addrs, ok := d.DNSCache.Get(name); ok {
			addrs1 := addrs.([]string)
			for _, addr := range addrs1 {
				seen[addr] = struct{}{}
			}
		}

		addrs := make([]string, 0)
		for addr, _ := range seen {
			addrs = append(addrs, addr)
		}

		d.DNSCache.Set(name, addrs, expire)
	}

	return nil
}
コード例 #10
0
ファイル: autoproxy.go プロジェクト: ChoyesYan/goproxy
func (f *Filter) Request(ctx context.Context, req *http.Request) (context.Context, *http.Request, error) {
	if strings.HasPrefix(req.RequestURI, "/") {
		return ctx, req, nil
	}

	host := req.Host
	if h, _, err := net.SplitHostPort(req.Host); err == nil {
		host = h
	}

	if f.SiteFiltersEnabled {
		if f1, ok := f.SiteFiltersRules.Lookup(host); ok {
			glog.V(2).Infof("%s \"AUTOPROXY SiteFilters %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
			filters.SetRoundTripFilter(ctx, f1.(filters.RoundTripFilter))
			return ctx, req, nil
		}
	}

	if f.RegionFiltersEnabled {
		if f1, ok := f.RegionFilterCache.Get(host); ok {
			filters.SetRoundTripFilter(ctx, f1.(filters.RoundTripFilter))
		} else if ips, err := net.LookupHost(host); err == nil {
			ip := ips[0]

			if strings.Contains(ip, ":") {
				if f1, ok := f.RegionFiltersRules["ipv6"]; ok {
					glog.V(2).Infof("%s \"AUTOPROXY RegionFilters IPv6 %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
					f.RegionFilterCache.Set(host, f1, time.Now().Add(time.Hour))
					filters.SetRoundTripFilter(ctx, f1)
				}
			} else if country, err := f.FindCountryByIP(ip); err == nil {
				if f1, ok := f.RegionFiltersRules[country]; ok {
					glog.V(2).Infof("%s \"AUTOPROXY RegionFilters %s %s %s %s\" with %T", req.RemoteAddr, country, req.Method, req.URL.String(), req.Proto, f1)
					f.RegionFilterCache.Set(host, f1, time.Now().Add(time.Hour))
					filters.SetRoundTripFilter(ctx, f1)
				} else if f1, ok := f.RegionFiltersRules["default"]; ok {
					glog.V(2).Infof("%s \"AUTOPROXY RegionFilters Default %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
					f.RegionFilterCache.Set(host, f1, time.Now().Add(time.Hour))
					filters.SetRoundTripFilter(ctx, f1)
				}
			}
		}
	}

	return ctx, req, nil
}
コード例 #11
0
ファイル: php.go プロジェクト: johnsonz/goproxy
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
	resp, err := f.Transport.RoundTrip(req)
	if err != nil {
		return ctx, nil, err
	} else {
		glog.V(2).Infof("%s \"PHP %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
	}
	return ctx, resp, nil
}
コード例 #12
0
ファイル: ratelimit.go プロジェクト: chenzhenjia/goproxy
func (f *Filter) Response(ctx context.Context, resp *http.Response) (context.Context, *http.Response, error) {

	if f.Rate > 0 && resp.ContentLength > f.Threshold {
		glog.V(2).Infof("RateLimit %#v rate to %#v", resp.Request.URL.String(), f.Rate)
		resp.Body = NewRateLimitReader(resp.Body, f.Rate, f.Capacity)
	}

	return ctx, resp, nil
}
コード例 #13
0
ファイル: multidialer.go プロジェクト: taiping-z/goproxy
func (d *MultiDialer) LookupAlias(alias string) (addrs []string, err error) {
	names, ok := d.HostMap[alias]
	if !ok {
		return nil, fmt.Errorf("alias %#v not exists", alias)
	}

	seen := make(map[string]struct{}, 0)
	expiry := time.Now().Add(d.DNSCacheExpiry)
	for _, name := range names {
		var addrs0 []string
		if net.ParseIP(name) != nil {
			addrs0 = []string{name}
			expiry = time.Time{}
		} else if addrs1, ok := d.DNSCache.Get(name); ok {
			addrs0 = addrs1.([]string)
		} else {
			if d.ForceIPv6 {
				addrs0, err = d.LookupHost2(name, d.DNSServers[0])
				if err != nil {
					glog.Warningf("LookupHost2(%#v, %#v) error: %s", name, d.DNSServers[0], err)
					addrs0 = []string{}
				}
			} else {
				addrs0, err = d.LookupHost(name)
				if err != nil {
					glog.Warningf("LookupHost(%#v) error: %s", name, err)
					addrs0 = []string{}
				}
			}

			glog.V(2).Infof("LookupHost(%#v) return %v", name, addrs0)
			d.DNSCache.Set(name, addrs0, expiry)
		}
		for _, addr := range addrs0 {
			seen[addr] = struct{}{}
		}
	}

	if len(seen) == 0 {
		return nil, err
	}

	addrs = make([]string, 0)
	for addr, _ := range seen {
		if _, ok := d.IPBlackList.GetQuiet(addr); ok {
			continue
		}
		addrs = append(addrs, addr)
	}

	if len(addrs) == 0 {
		glog.Errorf("MULTIDIALER: LookupAlias(%#v) have no good ip addrs", alias)
		return nil, fmt.Errorf("MULTIDIALER: LookupAlias(%#v) have no good ip addrs", alias)
	}

	return addrs, nil
}
コード例 #14
0
ファイル: autoproxy.go プロジェクト: ChoyesYan/goproxy
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
	if f := filters.GetRoundTripFilter(ctx); f != nil {
		return f.RoundTrip(ctx, req)
	}

	if req.URL.Host == "" && req.RequestURI[0] == '/' && f.IndexFilesEnabled {
		if _, ok := f.IndexFiles[req.URL.Path[1:]]; ok || req.URL.Path == "/" {
			if f.GFWListEnabled && strings.HasSuffix(req.URL.Path, ".pac") {
				glog.V(2).Infof("%s \"AUTOPROXY ProxyPac %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
				return f.ProxyPacRoundTrip(ctx, req)
			} else {
				glog.V(2).Infof("%s \"AUTOPROXY IndexFiles %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
				return f.IndexFilesRoundTrip(ctx, req)
			}
		}
	}

	return ctx, nil, nil
}
コード例 #15
0
ファイル: multidialer.go プロジェクト: taiping-z/goproxy
func (d *MultiDialer) dialMulti(network string, addrs []string) (net.Conn, error) {
	glog.V(3).Infof("dialMulti(%v, %v)", network, addrs)
	type racer struct {
		c net.Conn
		e error
	}

	length := len(addrs)
	if d.Level < length {
		length = d.Level
	}

	addrs = pickupAddrs(addrs, length, d.TCPConnDuration, d.TCPConnError)
	lane := make(chan racer, length)

	for _, addr := range addrs {
		go func(addr string, c chan<- racer) {
			start := time.Now()
			conn, err := d.Dialer.Dial(network, addr)
			end := time.Now()
			if err == nil {
				if d.TCPConnReadBuffer > 0 {
					if tc, ok := conn.(*net.TCPConn); ok {
						tc.SetReadBuffer(d.TCPConnReadBuffer)
					}
				}
				d.TCPConnDuration.Set(addr, end.Sub(start), end.Add(d.ConnExpiry))
			} else {
				d.TCPConnDuration.Del(addr)
				d.TCPConnError.Set(addr, err, end.Add(d.ConnExpiry))
			}
			lane <- racer{conn, err}
		}(addr, lane)
	}

	var r racer
	for i := 0; i < length; i++ {
		r = <-lane
		if r.e == nil {
			go func(count int) {
				var r1 racer
				for ; count > 0; count-- {
					r1 = <-lane
					if r1.c != nil {
						r1.c.Close()
					}
				}
			}(length - 1 - i)
			return r.c, nil
		}
	}
	return nil, r.e
}
コード例 #16
0
ファイル: resolver.go プロジェクト: johnsonz/goproxy
func (r *Resolver) LookupIP(name string) ([]net.IP, error) {
	if r.LRUCache != nil {
		if v, ok := r.LRUCache.Get(name); ok {
			switch v.(type) {
			case []net.IP:
				return v.([]net.IP), nil
			case string:
				return r.LookupIP(v.(string))
			default:
				return nil, fmt.Errorf("LookupIP: cannot convert %T(%+v) to []net.IP", v, v)
			}
		}
	}

	if ip := net.ParseIP(name); ip != nil {
		ips := []net.IP{ip}
		if r.LRUCache != nil {
			r.LRUCache.Set(name, ips, time.Time{})
		}
		return ips, nil
	}

	lookupIP := r.lookupIP1
	if r.DNSServer != nil {
		lookupIP = r.lookupIP2
	}

	ips, err := lookupIP(name)
	if err == nil {
		if r.BlackList != nil {
			ips1 := ips[:0]
			for _, ip := range ips {
				if _, ok := r.BlackList.GetQuiet(ip.String()); !ok {
					ips1 = append(ips1, ip)
				}
			}
			ips = ips1
		}

		if r.LRUCache != nil && len(ips) > 0 {
			if r.DNSExpiry == 0 {
				r.LRUCache.Set(name, ips, time.Time{})
			} else {
				r.LRUCache.Set(name, ips, time.Now().Add(r.DNSExpiry))
			}
		}
	}

	glog.V(2).Infof("LookupIP(%#v) return %+v, err=%+v", name, ips, err)
	return ips, err
}
コード例 #17
0
ファイル: autoproxy.go プロジェクト: phuslu/goagent
func (f *Filter) Request(ctx context.Context, req *http.Request) (context.Context, *http.Request, error) {
	if strings.HasPrefix(req.RequestURI, "/") {
		return ctx, req, nil
	}

	host := helpers.GetHostName(req)

	if f.BlackListEnabled {
		if f.BlackListSiteMatcher.Match(host) {
			glog.V(2).Infof("%s \"AUTOPROXY BlackList %s %s %s\"", req.RemoteAddr, req.Method, req.URL.String(), req.Proto)
			return ctx, filters.DummyRequest, nil
		}
	}

	if f.SiteFiltersEnabled {
		if f1, ok := f.SiteFiltersRules.Lookup(host); ok {
			glog.V(2).Infof("%s \"AUTOPROXY SiteFilters %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
			filters.SetRoundTripFilter(ctx, f1.(filters.RoundTripFilter))
			return ctx, req, nil
		}
	}

	if f.RegionFiltersEnabled {
		if f1, ok := f.RegionFilterCache.Get(host); ok {
			if f1 != nil {
				filters.SetRoundTripFilter(ctx, f1.(filters.RoundTripFilter))
			}
		} else if ips, err := f.RegionResolver.LookupIP(host); err == nil && len(ips) > 0 {
			ip := ips[0]

			if ip.IsLoopback() && !(strings.Contains(host, ".local") || strings.Contains(host, "localhost.")) {
				glog.V(2).Infof("%s \"AUTOPROXY RegionFilters BYPASS Loopback %s %s %s\" with nil", req.RemoteAddr, req.Method, req.URL.String(), req.Proto)
				f.RegionFilterCache.Set(host, nil, time.Now().Add(time.Hour))
			} else if ip.To4() == nil {
				if f1, ok := f.RegionFiltersRules["ipv6"]; ok {
					glog.V(2).Infof("%s \"AUTOPROXY RegionFilters IPv6 %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
					f.RegionFilterCache.Set(host, f1, time.Now().Add(time.Hour))
					filters.SetRoundTripFilter(ctx, f1)
				}
			} else if country, err := f.FindCountryByIP(ip.String()); err == nil {
				if f1, ok := f.RegionFiltersRules[country]; ok {
					glog.V(2).Infof("%s \"AUTOPROXY RegionFilters %s %s %s %s\" with %T", req.RemoteAddr, country, req.Method, req.URL.String(), req.Proto, f1)
					f.RegionFilterCache.Set(host, f1, time.Now().Add(time.Hour))
					filters.SetRoundTripFilter(ctx, f1)
				} else if f1, ok := f.RegionFiltersRules["default"]; ok {
					glog.V(2).Infof("%s \"AUTOPROXY RegionFilters Default %s %s %s\" with %T", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, f1)
					f.RegionFilterCache.Set(host, f1, time.Now().Add(time.Hour))
					filters.SetRoundTripFilter(ctx, f1)
				} else {
					f.RegionFilterCache.Set(host, nil, time.Now().Add(time.Hour))
				}
			}
		}
	}

	return ctx, req, nil
}
コード例 #18
0
ファイル: autoproxy.go プロジェクト: phuslu/goagent
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
	if f := filters.GetRoundTripFilter(ctx); f != nil {
		return f.RoundTrip(ctx, req)
	}

	switch {
	case f.SiteFiltersEnabled && req.URL.Scheme == "https":
		if f1, ok := f.SiteFiltersRules.Lookup(helpers.GetHostName(req)); ok && f1 != nil {
			return f1.(filters.RoundTripFilter).RoundTrip(ctx, req)
		}
	case f.RegionFiltersEnabled && req.URL.Scheme == "https":
		if f1, ok := f.RegionFilterCache.Get(helpers.GetHostName(req)); ok && f1 != nil {
			return f1.(filters.RoundTripFilter).RoundTrip(ctx, req)
		}
	}

	if req.URL.Host == "" && req.RequestURI[0] == '/' && f.IndexFilesEnabled {
		if _, ok := f.IndexFilesSet[req.URL.Path[1:]]; ok || req.URL.Path == "/" {
			switch {
			case f.GFWListEnabled && strings.HasSuffix(req.URL.Path, ".pac"):
				glog.V(2).Infof("%s \"AUTOPROXY ProxyPac %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
				return f.ProxyPacRoundTrip(ctx, req)
			case f.MobileConfigEnabled && strings.HasSuffix(req.URL.Path, ".mobileconfig"):
				glog.V(2).Infof("%s \"AUTOPROXY ProxyMobileConfig %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
				return f.ProxyMobileConfigRoundTrip(ctx, req)
			case f.IPHTMLEnabled && req.URL.Path == "/ip.html":
				glog.V(2).Infof("%s \"AUTOPROXY IPHTML %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
				return f.IPHTMLRoundTrip(ctx, req)
			default:
				glog.V(2).Infof("%s \"AUTOPROXY IndexFiles %s %s %s\" - -", req.RemoteAddr, req.Method, req.RequestURI, req.Proto)
				return f.IndexFilesRoundTrip(ctx, req)
			}
		}
	}

	return ctx, nil, nil
}
コード例 #19
0
ファイル: dialer.go プロジェクト: johnsonz/goproxy
func (d *Dialer) Dial(network, address string) (conn net.Conn, err error) {
	glog.V(3).Infof("Dail(%#v, %#v)", network, address)

	switch network {
	case "tcp", "tcp4", "tcp6":
		if d.Resolver != nil {
			if host, port, err := net.SplitHostPort(address); err == nil {
				if ips, err := d.Resolver.LookupIP(host); err == nil {
					if len(ips) == 0 {
						return nil, net.InvalidAddrError(fmt.Sprintf("Invaid DNS Record: %s", address))
					}
					return d.dialMulti(network, address, ips, port)
				}
			}
		}
	}

	return d.Dialer.Dial(network, address)
}
コード例 #20
0
ファイル: rootca.go プロジェクト: ChoyesYan/goproxy
func (c *RootCA) Issue(commonName string, vaildFor time.Duration, rsaBits int) (*tls.Certificate, error) {
	certFile := c.toFilename(commonName, ".crt")

	if storage.IsNotExist(c.store, certFile) {
		glog.V(2).Infof("Issue %s certificate for %#v...", c.name, commonName)
		c.mu.Lock()
		defer c.mu.Unlock()
		if storage.IsNotExist(c.store, certFile) {
			if err := c.issue(commonName, vaildFor, rsaBits); err != nil {
				return nil, err
			}
		}
	}

	tlsCert, err := tls.LoadX509KeyPair(certFile, certFile)
	if err != nil {
		return nil, err
	}
	return &tlsCert, nil
}
コード例 #21
0
ファイル: dialer.go プロジェクト: chenzhenjia/goproxy
func (d *Dialer) Dial(network, address string) (conn net.Conn, err error) {
	glog.V(3).Infof("Dail(%#v, %#v)", network, address)

	switch network {
	case "tcp", "tcp4", "tcp6":
		if d.DNSCache != nil {
			if addr, ok := d.DNSCache.Get(address); ok {
				address = addr.(string)
			} else {
				if host, port, err := net.SplitHostPort(address); err == nil {
					if ips, err := net.LookupIP(host); err == nil && len(ips) > 0 {
						ip := ips[0].String()
						if d.LoopbackAddrs != nil {
							if _, ok := d.LoopbackAddrs[ip]; ok {
								return nil, net.InvalidAddrError(fmt.Sprintf("Invaid DNS Record: %s(%s)", host, ip))
							}
						}
						addr := net.JoinHostPort(ip, port)
						expiry := d.DNSCacheExpiry
						if expiry == 0 {
							expiry = DefaultDNSCacheExpiry
						}
						d.DNSCache.Set(address, addr, time.Now().Add(expiry))
						glog.V(3).Infof("direct Dial cache dns %#v=%#v", address, addr)
						address = addr
					}
				}
			}
		}
	default:
		break
	}

	if d.Level <= 1 {
		retry := d.RetryTimes
		if retry == 0 {
			retry = DefaultRetryTimes
		}

		for i := 0; i < retry; i++ {
			conn, err = d.Dialer.Dial(network, address)
			if err == nil || i == retry-1 {
				break
			}
			retryDelay := d.RetryDelay
			if retryDelay == 0 {
				retryDelay = DefaultRetryDelay
			}
			time.Sleep(retryDelay)
		}
		return conn, err
	} else {
		type racer struct {
			c net.Conn
			e error
		}

		lane := make(chan racer, d.Level)
		retry := (d.RetryTimes + d.Level - 1) / d.Level
		for i := 0; i < retry; i++ {
			for j := 0; j < d.Level; j++ {
				go func(addr string, c chan<- racer) {
					conn, err := d.Dialer.Dial(network, addr)
					lane <- racer{conn, err}
				}(address, lane)
			}

			var r racer
			for k := 0; k < d.Level; k++ {
				r = <-lane
				if r.e == nil {
					go func(count int) {
						var r1 racer
						for ; count > 0; count-- {
							r1 = <-lane
							if r1.c != nil {
								r1.c.Close()
							}
						}
					}(d.Level - 1 - k)
					return r.c, nil
				}
			}

			if i == retry-1 {
				return nil, r.e
			}
		}
	}

	return nil, net.UnknownNetworkError("Unkown transport/direct error")
}
コード例 #22
0
ファイル: gaetransport.go プロジェクト: ChoyesYan/goproxy
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
	for i := 0; i < t.RetryTimes; i++ {
		server := t.Servers.PickFetchServer(req, i)
		req1, err := t.Servers.EncodeRequest(req, server)
		if err != nil {
			return nil, fmt.Errorf("GAE EncodeRequest: %s", err.Error())
		}

		resp, err := t.RoundTripper.RoundTrip(req1)

		if err != nil {

			isTimeoutError := false
			if ne, ok := err.(interface {
				Timeout() bool
			}); ok && ne.Timeout() {
				isTimeoutError = true
			}
			if ne, ok := err.(*net.OpError); ok && ne.Op == "read" {
				isTimeoutError = true
			}

			if isTimeoutError {
				glog.Warningf("GAE: \"%s %s\" timeout: %v, helpers.TryCloseConnections(%T)", req.Method, req.URL.String(), err, t.RoundTripper)
				helpers.TryCloseConnections(t.RoundTripper)
			}

			if i == t.RetryTimes-1 {
				return nil, err
			} else {
				glog.Warningf("GAE: request \"%s\" error: %T(%v), retry...", req.URL.String(), err, err)
				continue
			}
		}

		if resp.StatusCode != http.StatusOK {
			if i == t.RetryTimes-1 {
				return resp, nil
			}

			switch resp.StatusCode {
			case http.StatusServiceUnavailable:
				glog.Warningf("GAE: %s over qouta, try switch to next appid...", server.Host)
				t.Servers.ToggleBadServer(server)
				time.Sleep(t.RetryDelay)
				continue
			case http.StatusFound,
				http.StatusBadGateway,
				http.StatusNotFound,
				http.StatusMethodNotAllowed:
				if t.MultiDialer != nil {
					if addr, err := helpers.ReflectRemoteAddrFromResponse(resp); err == nil {
						if ip, _, err := net.SplitHostPort(addr); err == nil {
							duration := 8 * time.Hour
							glog.Warningf("GAE: %s StatusCode is %d, not a gws/gvs ip, add to blacklist for %v", ip, resp.StatusCode, duration)
							t.MultiDialer.IPBlackList.Set(ip, struct{}{}, time.Now().Add(duration))
						}
						if ok := helpers.TryCloseConnectionByRemoteAddr(t.RoundTripper, addr); !ok {
							glog.Warningf("GAE: TryCloseConnectionByRemoteAddr(%T, %#v) failed.", t.RoundTripper, addr)
						}
					}
				}
				continue
			default:
				return resp, nil
			}
		}

		resp1, err := t.Servers.DecodeResponse(resp)
		if err != nil {
			return nil, err
		}
		if resp1 != nil {
			resp1.Request = req
		}
		if i == t.RetryTimes-1 {
			return resp, err
		}

		switch resp1.StatusCode {
		case http.StatusBadGateway:
			body, err := ioutil.ReadAll(resp1.Body)
			if err != nil {
				resp1.Body.Close()
				return nil, err
			}
			resp1.Body.Close()
			switch {
			case bytes.Contains(body, []byte("DEADLINE_EXCEEDED")):
				glog.V(2).Infof("GAE: %s urlfetch %#v get DEADLINE_EXCEEDED, retry...", req1.Host, req.URL.String())
				continue
			case bytes.Contains(body, []byte("ver quota")):
				glog.V(2).Infof("GAE: %s urlfetch %#v get over quota, retry...", req1.Host, req.URL.String())
				time.Sleep(t.RetryDelay)
				continue
			case bytes.Contains(body, []byte("urlfetch: CLOSED")):
				glog.V(2).Infof("GAE: %s urlfetch %#v get urlfetch: CLOSED, retry...", req1.Host, req.URL.String())
				time.Sleep(t.RetryDelay)
				continue
			default:
				resp1.Body = ioutil.NopCloser(bytes.NewReader(body))
				return resp1, nil
			}
		default:
			return resp1, nil
		}
	}

	return nil, fmt.Errorf("GAE: cannot reach here with %#v", req)
}
コード例 #23
0
ファイル: gae.go プロジェクト: ChoyesYan/goproxy
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
	if !f.SiteMatcher.Match(req.Host) {
		return ctx, nil, nil
	}

	var tr http.RoundTripper = f.GAETransport

	if req.URL.Scheme == "http" && f.ForceHTTPSMatcher.Match(req.Host) {
		if !strings.HasPrefix(req.Header.Get("Referer"), "https://") {
			u := strings.Replace(req.URL.String(), "http://", "https://", 1)
			glog.V(2).Infof("GAE FORCEHTTPS get raw url=%v, redirect to %v", req.URL.String(), u)
			resp := &http.Response{
				StatusCode: http.StatusMovedPermanently,
				Header: http.Header{
					"Location": []string{u},
				},
				Request:       req,
				Close:         true,
				ContentLength: -1,
			}
			glog.V(2).Infof("%s \"GAE FORCEHTTPS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
			return ctx, resp, nil
		}
	}

	if f.DirectSiteMatcher.Match(req.Host) {
		if req.URL.Path == "/url" {
			if rawurl := req.URL.Query().Get("url"); rawurl != "" {
				if u, err := url.Parse(rawurl); err == nil {
					if u.Scheme == "http" && f.ForceHTTPSMatcher.Match(u.Host) {
						rawurl = strings.Replace(rawurl, "http://", "https://", 1)
					}
				}
				glog.V(2).Infof("%s \"GAE REDIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, rawurl, req.Proto)
				return ctx, &http.Response{
					StatusCode: http.StatusFound,
					Header: http.Header{
						"Location": []string{rawurl},
					},
					Request:       req,
					Close:         true,
					ContentLength: -1,
				}, nil
			}
		}

		if req.URL.Scheme != "http" && !f.shouldForceGAE(req) {
			tr = f.DirectTransport
			if s := req.Header.Get("Connection"); s != "" {
				if s1 := strings.ToLower(s); s != s1 {
					req.Header.Set("Connection", s1)
				}
			}
		}
	}

	if tr != f.DirectTransport && req.Method == http.MethodOptions {
		if v, ok := f.FakeOptionsMatcher.Lookup(req.Host); ok {
			resp := &http.Response{
				StatusCode:    http.StatusOK,
				Header:        http.Header{},
				Request:       req,
				Close:         false,
				ContentLength: -1,
			}
			for _, s := range v.([]string) {
				parts := strings.SplitN(s, ":", 2)
				if len(parts) == 2 {
					resp.Header.Add(parts[0], strings.TrimSpace(parts[1]))
				}
			}
			if origin := req.Header.Get("Origin"); origin != "" {
				resp.Header.Set("Access-Control-Allow-Origin", origin)
			}
			if headers := req.Header.Get("Access-Control-Request-Headers"); headers != "" {
				resp.Header.Set("Access-Control-Allow-Headers", headers)
			}
			glog.V(2).Infof("%s \"GAE FAKEOPTIONS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
			return ctx, resp, nil
		}
	}

	prefix := "FETCH"
	if tr == f.DirectTransport {
		prefix = "DIRECT"
	}

	resp, err := tr.RoundTrip(req)
	if err != nil {
		glog.Warningf("%s \"GAE %s %s %s %s\" error: %T(%v)", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, err, err)
		if tr == f.DirectTransport {
			if ne, ok := err.(interface {
				Timeout() bool
			}); ok && ne.Timeout() {
				// f.MultiDialer.ClearCache()
				helpers.TryCloseConnections(tr)
			}
		}
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
		return ctx, nil, err
	}

	if strings.HasPrefix(resp.Header.Get("Content-Type"), "text/") &&
		resp.Header.Get("Content-Encoding") == "" &&
		f.ForceDeflateMatcher.Match(req.Host) {
		buf := make([]byte, 1024)
		n, err := resp.Body.Read(buf)
		if err != nil {
			defer resp.Body.Close()
			return ctx, nil, err
		}
		buf = buf[:n]
		switch {
		case helpers.IsGzip(buf):
			resp.Header.Set("Content-Encoding", "gzip")
		case helpers.IsBinary(buf):
			resp.Header.Set("Content-Encoding", "deflate")
		}
		resp.Body = helpers.NewMultiReadCloser(bytes.NewReader(buf), resp.Body)
	}

	if resp != nil && resp.Header != nil {
		resp.Header.Del("Alt-Svc")
		resp.Header.Del("Alternate-Protocol")
	}

	glog.V(2).Infof("%s \"GAE %s %s %s %s\" %d %s", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
	return ctx, resp, err
}
コード例 #24
0
ファイル: gae.go プロジェクト: ChoyesYan/goproxy
func NewFilter(config *Config) (filters.Filter, error) {
	dnsServers := make([]net.IP, 0)
	for _, s := range config.DNSServers {
		if ip := net.ParseIP(s); ip != nil {
			dnsServers = append(dnsServers, ip)
		}
	}

	GoogleG2PKP, err := base64.StdEncoding.DecodeString(config.GoogleG2PKP)
	if err != nil {
		return nil, err
	}

	googleTLSConfig := &tls.Config{
		MinVersion:         tls.VersionTLS12,
		InsecureSkipVerify: true,
		ServerName:         "www.microsoft.com",
		ClientSessionCache: tls.NewLRUClientSessionCache(config.TLSConfig.ClientSessionCacheSize),
		CipherSuites: []uint16{
			tls.TLS_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
		},
	}
	switch config.TLSConfig.Version {
	case "TLSv12", "TLSv1.2":
		googleTLSConfig.MinVersion = tls.VersionTLS12
	case "TLSv11", "TLSv1.1":
		googleTLSConfig.MinVersion = tls.VersionTLS11
	default:
		googleTLSConfig.MinVersion = tls.VersionTLS10
	}
	pickupCiphers := func(names []string) []uint16 {
		ciphers := make([]uint16, 0)
		for _, name := range names {
			cipher := helpers.Cipher(name)
			if cipher == 0 {
				glog.Fatalf("GAE: cipher %#v is not supported.", name)
			}
			ciphers = append(ciphers, cipher)
		}
		helpers.ShuffleUint16s(ciphers)
		ciphers = ciphers[:1+rand.Intn(len(ciphers))]
		ciphers1 := []uint16{}
		for _, name := range []string{
			"TLS_RSA_WITH_AES_256_CBC_SHA256",
			"TLS_RSA_WITH_AES_256_GCM_SHA384",
			"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
			"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
		} {
			if !helpers.ContainsString(names, name) {
				if c := helpers.Cipher(name); c != 0 {
					ciphers1 = append(ciphers1, c)
				}
			}
		}
		helpers.ShuffleUint16s(ciphers1)
		ciphers1 = ciphers1[:rand.Intn(len(ciphers1))]
		ciphers = append(ciphers, ciphers1...)
		helpers.ShuffleUint16s(ciphers)
		return ciphers
	}
	googleTLSConfig.CipherSuites = pickupCiphers(config.TLSConfig.Ciphers)
	if len(config.TLSConfig.ServerName) > 0 {
		googleTLSConfig.ServerName = config.TLSConfig.ServerName[rand.Intn(len(config.TLSConfig.ServerName))]
	}
	if !config.DisableHTTP2 {
		googleTLSConfig.NextProtos = []string{"h2", "h2-14", "http/1.1"}
	}

	if config.Site2Alias == nil {
		config.Site2Alias = make(map[string]string)
	}
	for key, value := range config.SiteToAlias {
		config.Site2Alias[key] = value
	}
	config.SiteToAlias = config.Site2Alias

	hostmap := map[string][]string{}
	for key, value := range config.HostMap {
		hostmap[key] = helpers.UniqueStrings(value)
	}

	d := &net.Dialer{
		KeepAlive: time.Duration(config.Transport.Dialer.KeepAlive) * time.Second,
		Timeout:   time.Duration(config.Transport.Dialer.Timeout) * time.Second,
		DualStack: config.Transport.Dialer.DualStack,
	}

	md := &dialer.MultiDialer{
		Dialer:            d,
		DisableIPv6:       config.DisableIPv6,
		ForceIPv6:         config.ForceIPv6,
		SSLVerify:         config.SSLVerify,
		EnableRemoteDNS:   config.EnableRemoteDNS,
		LogToStderr:       flag.Lookup("logtostderr") != nil,
		TLSConfig:         nil,
		SiteToAlias:       helpers.NewHostMatcherWithString(config.SiteToAlias),
		IPBlackList:       lrucache.NewLRUCache(1024),
		HostMap:           hostmap,
		GoogleTLSConfig:   googleTLSConfig,
		GoogleG2PKP:       GoogleG2PKP,
		DNSServers:        dnsServers,
		DNSCache:          lrucache.NewLRUCache(config.Transport.Dialer.DNSCacheSize),
		DNSCacheExpiry:    time.Duration(config.Transport.Dialer.DNSCacheExpiry) * time.Second,
		TLSConnDuration:   lrucache.NewLRUCache(8192),
		TLSConnError:      lrucache.NewLRUCache(8192),
		TLSConnReadBuffer: config.Transport.Dialer.SocketReadBuffer,
		ConnExpiry:        5 * time.Minute,
		Level:             config.Transport.Dialer.Level,
	}

	for _, ip := range config.IPBlackList {
		md.IPBlackList.Set(ip, struct{}{}, time.Time{})
	}

	var tr http.RoundTripper

	t1 := &http.Transport{
		Dial:                  md.Dial,
		DialTLS:               md.DialTLS,
		DisableKeepAlives:     config.Transport.DisableKeepAlives,
		DisableCompression:    config.Transport.DisableCompression,
		ResponseHeaderTimeout: time.Duration(config.Transport.ResponseHeaderTimeout) * time.Second,
		IdleConnTimeout:       time.Duration(config.Transport.IdleConnTimeout) * time.Second,
		MaxIdleConnsPerHost:   config.Transport.MaxIdleConnsPerHost,
	}

	if config.Transport.Proxy.Enabled {
		fixedURL, err := url.Parse(config.Transport.Proxy.URL)
		if err != nil {
			glog.Fatalf("url.Parse(%#v) error: %s", config.Transport.Proxy.URL, err)
		}

		dialer, err := proxy.FromURL(fixedURL, d, &dialer.MultiResolver{md})
		if err != nil {
			glog.Fatalf("proxy.FromURL(%#v) error: %s", fixedURL.String(), err)
		}

		t1.Dial = dialer.Dial
		t1.DialTLS = nil
		t1.Proxy = nil
		t1.TLSClientConfig = md.GoogleTLSConfig
	}

	switch {
	case config.DisableHTTP2 && config.ForceHTTP2:
		glog.Fatalf("GAE: DisableHTTP2=%v and ForceHTTP2=%v is conflict!", config.DisableHTTP2, config.ForceHTTP2)
	case config.Transport.Proxy.Enabled && config.ForceHTTP2:
		glog.Fatalf("GAE: Proxy.Enabled=%v and ForceHTTP2=%v is conflict!", config.Transport.Proxy.Enabled, config.ForceHTTP2)
	case config.ForceHTTP2:
		tr = &http2.Transport{
			DialTLS:            md.DialTLS2,
			TLSClientConfig:    md.GoogleTLSConfig,
			DisableCompression: config.Transport.DisableCompression,
		}
	case !config.DisableHTTP2:
		err := http2.ConfigureTransport(t1)
		if err != nil {
			glog.Warningf("GAE: Error enabling Transport HTTP/2 support: %v", err)
		}
		tr = t1
	default:
		tr = t1
	}

	forceHTTPSMatcherStrings := make([]string, 0)
	for key, value := range config.SiteToAlias {
		if strings.HasPrefix(value, "google_") {
			forceHTTPSMatcherStrings = append(forceHTTPSMatcherStrings, key)
		}
	}

	forceGAEStrings := make([]string, 0)
	forceGAESuffixs := make([]string, 0)
	forceGAEMatcherStrings := make([]string, 0)
	for _, s := range config.ForceGAE {
		if strings.Contains(s, "/") {
			if strings.HasSuffix(s, "$") {
				forceGAESuffixs = append(forceGAESuffixs, strings.TrimRight(s, "$"))
			} else {
				forceGAEStrings = append(forceGAEStrings, s)
			}
		} else {
			forceGAEMatcherStrings = append(forceGAEMatcherStrings, s)
		}
	}

	if config.EnableDeadProbe && !config.Transport.Proxy.Enabled {
		go func() {
			probe := func() {
				req, _ := http.NewRequest(http.MethodGet, "https://clients3.google.com/generate_204", nil)
				ctx, cancel := context.WithTimeout(req.Context(), 3*time.Second)
				defer cancel()
				req = req.WithContext(ctx)
				resp, err := tr.RoundTrip(req)
				if resp != nil && resp.Body != nil {
					glog.V(3).Infof("GAE EnableDeadProbe \"%s %s\" %d -", req.Method, req.URL.String(), resp.StatusCode)
					resp.Body.Close()
				}
				if err != nil {
					glog.V(2).Infof("GAE EnableDeadProbe \"%s %s\" error: %v", req.Method, req.URL.String(), err)
					s := strings.ToLower(err.Error())
					if strings.HasPrefix(s, "net/http: request canceled") || strings.Contains(s, "timeout") {
						helpers.TryCloseConnections(tr)
					}
				}
			}

			for {
				time.Sleep(time.Duration(5+rand.Intn(6)) * time.Second)
				probe()
			}
		}()
	}

	helpers.ShuffleStrings(config.AppIDs)

	f := &Filter{
		Config: *config,
		GAETransport: &Transport{
			RoundTripper: tr,
			MultiDialer:  md,
			Servers: NewServers(config.AppIDs,
				config.Password,
				config.SSLVerify,
				time.Duration(config.Transport.ResponseHeaderTimeout-4)*time.Second),
			RetryDelay: time.Duration(config.Transport.RetryDelay*1000) * time.Second,
			RetryTimes: config.Transport.RetryTimes,
		},
		DirectTransport:     tr,
		ForceHTTPSMatcher:   helpers.NewHostMatcher(forceHTTPSMatcherStrings),
		ForceGAEMatcher:     helpers.NewHostMatcher(forceGAEMatcherStrings),
		ForceGAEStrings:     forceGAEStrings,
		ForceGAESuffixs:     forceGAESuffixs,
		ForceDeflateMatcher: helpers.NewHostMatcher(config.ForceDeflate),
		FakeOptionsMatcher:  helpers.NewHostMatcherWithStrings(config.FakeOptions),
		SiteMatcher:         helpers.NewHostMatcher(config.Sites),
		DirectSiteMatcher:   helpers.NewHostMatcherWithString(config.Site2Alias),
	}

	if config.Transport.Proxy.Enabled {
		f.GAETransport.MultiDialer = nil
	}

	return f, nil
}
コード例 #25
0
ファイル: gae.go プロジェクト: johnsonz/goproxy
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
	var tr http.RoundTripper = f.GAETransport

	if req.URL.Scheme == "http" && f.ForceHTTPSMatcher.Match(req.Host) {
		if !strings.HasPrefix(req.Header.Get("Referer"), "https://") {
			u := strings.Replace(req.URL.String(), "http://", "https://", 1)
			glog.V(2).Infof("GAE FORCEHTTPS get raw url=%v, redirect to %v", req.URL.String(), u)
			resp := &http.Response{
				StatusCode: http.StatusMovedPermanently,
				Header: http.Header{
					"Location": []string{u},
				},
				Request:       req,
				Close:         true,
				ContentLength: -1,
			}
			glog.V(2).Infof("%s \"GAE FORCEHTTPS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
			return ctx, resp, nil
		}
	}

	if f.DirectSiteMatcher.Match(req.Host) {
		switch req.URL.Path {
		case "/url":
			if rawurl := req.URL.Query().Get("url"); rawurl != "" {
				if u, err := url.Parse(rawurl); err == nil {
					if u.Scheme == "http" && f.ForceHTTPSMatcher.Match(u.Host) {
						rawurl = strings.Replace(rawurl, "http://", "https://", 1)
					}
				}
				glog.V(2).Infof("%s \"GAE REDIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, rawurl, req.Proto)
				return ctx, &http.Response{
					StatusCode: http.StatusFound,
					Header: http.Header{
						"Location": []string{rawurl},
					},
					Request:       req,
					Close:         true,
					ContentLength: -1,
				}, nil
			}
		case "/books":
			if req.URL.Host == "books.google.cn" {
				rawurl := strings.Replace(req.URL.String(), "books.google.cn", "books.google.com", 1)
				glog.V(2).Infof("%s \"GAE REDIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, rawurl, req.Proto)
				return ctx, &http.Response{
					StatusCode: http.StatusFound,
					Header: http.Header{
						"Location": []string{rawurl},
					},
					Request:       req,
					Close:         true,
					ContentLength: -1,
				}, nil
			}
		}

		if req.URL.Scheme != "http" && !f.shouldForceGAE(req) {
			tr = f.DirectTransport
			if s := req.Header.Get("Connection"); s != "" {
				if s1 := strings.ToLower(s); s != s1 {
					req.Header.Set("Connection", s1)
				}
			}
		}
	}

	if tr != f.DirectTransport && req.Method == http.MethodOptions {
		if v, ok := f.FakeOptionsMatcher.Lookup(req.Host); ok {
			resp := &http.Response{
				StatusCode:    http.StatusOK,
				Header:        http.Header{},
				Request:       req,
				Close:         false,
				ContentLength: -1,
			}
			for _, s := range v.([]string) {
				parts := strings.SplitN(s, ":", 2)
				if len(parts) == 2 {
					resp.Header.Add(parts[0], strings.TrimSpace(parts[1]))
				}
			}
			if origin := req.Header.Get("Origin"); origin != "" {
				resp.Header.Set("Access-Control-Allow-Origin", origin)
			}
			if headers := req.Header.Get("Access-Control-Request-Headers"); headers != "" {
				resp.Header.Set("Access-Control-Allow-Headers", headers)
			}
			glog.V(2).Infof("%s \"GAE FAKEOPTIONS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
			return ctx, resp, nil
		}
	}

	if tr == f.GAETransport {
		switch strings.ToLower(req.Header.Get("Connection")) {
		case "upgrade":
			resp := &http.Response{
				StatusCode: http.StatusForbidden,
				Header: http.Header{
					"X-WebSocket-Reject-Reason": []string{"Unsupported"},
				},
				Request:       req,
				Close:         true,
				ContentLength: 0,
			}
			glog.V(2).Infof("%s \"GAE FAKEWEBSOCKET %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
			return ctx, resp, nil
		}
	}

	prefix := "FETCH"
	if tr == f.DirectTransport {
		prefix = "DIRECT"
	}

	resp, err := tr.RoundTrip(req)
	if err != nil {
		glog.Warningf("%s \"GAE %s %s %s %s\" error: %T(%v)", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, err, err)
		if tr == f.DirectTransport {
			if ne, ok := err.(interface {
				Timeout() bool
			}); ok && ne.Timeout() {
				// f.MultiDialer.ClearCache()
				helpers.CloseConnections(tr)
			}
		}
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
		return ctx, nil, err
	}

	if tr == f.DirectTransport && resp.StatusCode >= http.StatusBadRequest {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			resp.Body.Close()
			return ctx, nil, err
		}
		if addr, err := helpers.ReflectRemoteAddrFromResponse(resp); err == nil {
			if ip, _, err := net.SplitHostPort(addr); err == nil {
				var duration time.Duration
				switch {
				case resp.StatusCode == http.StatusBadGateway && bytes.Contains(body, []byte("Please try again in 30 seconds.")):
					duration = 1 * time.Hour
				case resp.StatusCode == http.StatusNotFound && bytes.Contains(body, []byte("<ins>That’s all we know.</ins>")):
					server := resp.Header.Get("Server")
					if server != "gws" && !strings.HasPrefix(server, "gvs") {
						if md := f.GAETransport.MultiDialer; md != nil && md.TLSConnDuration.Len() > 10 {
							duration = 5 * time.Minute
						}
					}
				}

				if duration > 0 && f.GAETransport.MultiDialer != nil {
					glog.Warningf("GAE: %s StatusCode is %d, not a gws/gvs ip, add to blacklist for %v", ip, resp.StatusCode, duration)
					f.GAETransport.MultiDialer.IPBlackList.Set(ip, struct{}{}, time.Now().Add(duration))

					if !helpers.CloseConnectionByRemoteAddr(tr, addr) {
						glog.Warningf("GAE: CloseConnectionByRemoteAddr(%T, %#v) failed.", tr, addr)
					}
				}
			}
		}
		resp.Body.Close()
		resp.Body = ioutil.NopCloser(bytes.NewReader(body))
	}

	if resp != nil && resp.Header != nil {
		resp.Header.Del("Alt-Svc")
		resp.Header.Del("Alternate-Protocol")
	}

	glog.V(2).Infof("%s \"GAE %s %s %s %s\" %d %s", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
	return ctx, resp, err
}
コード例 #26
0
ファイル: dialer2.go プロジェクト: phuslu/goagent
func (d *MultiDialer) DialTLS2(network, address string, cfg *tls.Config) (net.Conn, error) {
	if d.LogToStderr {
		SetConsoleTextColorGreen()
	}
	glog.V(2).Infof("MULTIDIALER DialTLS2(%#v, %#v) with good_addrs=%d, bad_addrs=%d", network, address, d.TLSConnDuration.Len(), d.TLSConnError.Len())
	if d.LogToStderr {
		SetConsoleTextColorReset()
	}

	if cfg == nil {
		cfg = d.TLSConfig
	}

	switch network {
	case "tcp", "tcp4", "tcp6":
		if host, port, err := net.SplitHostPort(address); err == nil {
			if alias0, ok := d.SiteToAlias.Lookup(host); ok {
				alias := alias0.(string)
				if hosts, err := d.LookupAlias(alias); err == nil {
					var config *tls.Config

					isGoogleAddr := false
					switch {
					case strings.HasPrefix(alias, "google_"):
						config = d.GoogleTLSConfig
						isGoogleAddr = true
					case cfg == nil:
						config = &tls.Config{
							InsecureSkipVerify: !d.SSLVerify,
							ServerName:         address,
						}
					default:
						config = cfg
					}
					glog.V(3).Infof("DialTLS2(%#v, %#v) alais=%#v set tls.Config=%#v", network, address, alias, config)

					addrs := make([]string, len(hosts))
					for i, host := range hosts {
						addrs[i] = net.JoinHostPort(host, port)
					}
					switch {
					case d.Resolver.ForceIPv6:
						network = "tcp6"
					case d.Resolver.DisableIPv6:
						network = "tcp4"
					}
					conn, err := d.dialMultiTLS(network, addrs, config)
					if err != nil {
						return nil, err
					}
					if d.SSLVerify && isGoogleAddr {
						if tc, ok := conn.(*tls.Conn); ok {
							certs := tc.ConnectionState().PeerCertificates
							if len(tc.ConnectionState().PeerCertificates) <= 1 {
								return nil, fmt.Errorf("Wrong certificate of %s: PeerCertificates=%#v", conn.RemoteAddr(), certs)
							}
							cert := certs[1]
							glog.V(3).Infof("MULTIDIALER DialTLS(%#v, %#v) verify cert=%v", network, address, cert.Subject)
							if d.GoogleG2PKP != nil {
								pkp := sha256.Sum256(cert.RawSubjectPublicKeyInfo)
								if !bytes.Equal(pkp[:], d.GoogleG2PKP) {
									defer conn.Close()
									return nil, fmt.Errorf("Wrong certificate of %s: Issuer=%v, SubjectKeyId=%#v", conn.RemoteAddr(), cert.Subject, cert.SubjectKeyId)
								}
							} else {
								if !strings.HasPrefix(cert.Subject.CommonName, "Google ") {
									defer conn.Close()
									return nil, fmt.Errorf("Wrong certificate of %s: Issuer=%v, SubjectKeyId=%#v", conn.RemoteAddr(), cert.Subject, cert.SubjectKeyId)
								}
							}
						}
					}
					return conn, nil
				}
			}
		}
	default:
		break
	}

	if dialer, ok := d.Dialer.(*net.Dialer); ok {
		return tls.DialWithDialer(dialer, network, address, d.TLSConfig)
	} else {
		return tls.Dial(network, address, d.TLSConfig)
	}
}
コード例 #27
0
ファイル: autoproxy.go プロジェクト: chenzhenjia/goproxy
func (f *Filter) ProxyPacRoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
	filename := req.URL.Path[1:]

	data := ""

	obj, err := f.Store.GetObject(filename, -1, -1)
	switch {
	case os.IsNotExist(err):
		glog.V(2).Infof("AUTOPROXY ProxyPac: generate %#v", filename)

		s := fmt.Sprintf(`// User-defined FindProxyForURL
function FindProxyForURL(url, host) {
    if (shExpMatch(host, '*.google*.*') ||
       dnsDomainIs(host, '.ggpht.com') ||
       dnsDomainIs(host, '.gstatic.com') ||
       host == 'goo.gl') {
        return 'PROXY %s';
    }
    return 'DIRECT';
}
`, req.Host)
		f.Store.PutObject(filename, http.Header{}, ioutil.NopCloser(bytes.NewBufferString(s)))
	case err != nil:
		return ctx, nil, err
	case obj != nil:
		if body := obj.Body(); body != nil {
			body.Close()
		}
	}

	if obj, err := f.Store.GetObject(filename, -1, -1); err == nil {
		body := obj.Body()
		defer body.Close()
		if b, err := ioutil.ReadAll(body); err == nil {
			s := strings.Replace(string(b), "function FindProxyForURL(", "function MyFindProxyForURL(", 1)
			host, _, err := net.SplitHostPort(req.Host)
			if err != nil {
				host = req.Host
			}
			for _, localaddr := range []string{"127.0.0.1:", "[::1]:", "localhost:"} {
				s = strings.Replace(s, localaddr, net.JoinHostPort(host, ""), -1)
			}
			data += s
		}
	}

	if f.GFWListEnabled {
		data += f.AutoProxy2Pac.GeneratePac(req)
	} else {
		data += `
function FindProxyForURL(url, host) {
    if (isPlainHostName(host) ||
        host.indexOf('127.') == 0 ||
        host.indexOf('192.168.') == 0 ||
        host.indexOf('10.') == 0 ||
        shExpMatch(host, 'localhost.*')) {
        return 'DIRECT';
    }

    return MyFindProxyForURL(url, host);
}`
	}

	resp := &http.Response{
		StatusCode:    http.StatusOK,
		Header:        http.Header{},
		Request:       req,
		Close:         true,
		ContentLength: int64(len(data)),
		Body:          ioutil.NopCloser(bytes.NewReader([]byte(data))),
	}

	return ctx, resp, nil
}
コード例 #28
0
ファイル: autoproxypac.go プロジェクト: ChoyesYan/goproxy
func (f *Filter) ProxyPacRoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
	_, port, err := net.SplitHostPort(req.Host)
	if err != nil {
		port = "80"
	}

	if v, ok := f.ProxyPacCache.Get(req.RequestURI); ok {
		if s, ok := v.(string); ok {
			s = fixProxyPac(s, req)
			return ctx, &http.Response{
				StatusCode:    http.StatusOK,
				Header:        http.Header{},
				Request:       req,
				Close:         true,
				ContentLength: int64(len(s)),
				Body:          ioutil.NopCloser(strings.NewReader(s)),
			}, nil
		}
	}

	filename := req.URL.Path[1:]

	buf := new(bytes.Buffer)

	resp, err := f.Store.Get(filename, -1, -1)
	switch {
	case os.IsNotExist(err), resp.StatusCode == http.StatusNotFound:
		glog.V(2).Infof("AUTOPROXY ProxyPac: generate %#v", filename)
		s := fmt.Sprintf(`// User-defined FindProxyForURL
function FindProxyForURL(url, host) {
    if (isPlainHostName(host) ||
        host.indexOf('127.') == 0 ||
        host.indexOf('192.168.') == 0 ||
        host.indexOf('10.') == 0 ||
        shExpMatch(host, 'localhost.*')) {
        return 'DIRECT';
    }

    if (shExpMatch(host, '*.google*.*')) {
        return 'PROXY localhost:%s';
    }

    return 'DIRECT';
}
`, port)
		f.Store.Put(filename, http.Header{}, ioutil.NopCloser(bytes.NewBufferString(s)))
	case err != nil:
		return ctx, nil, err
	case resp.Body != nil:
		resp.Body.Close()
	}

	if resp, err := f.Store.Get(filename, -1, -1); err == nil {
		defer resp.Body.Close()
		if b, err := ioutil.ReadAll(resp.Body); err == nil {
			if f.GFWListEnabled {
				b = []byte(strings.Replace(string(b), "function FindProxyForURL(", "function MyFindProxyForURL(", 1))
			}
			buf.Write(b)
		}
	}

	if f.GFWListEnabled {
		resp, err := f.Store.Get(f.GFWList.Filename, -1, -1)
		if err != nil {
			glog.Errorf("GetObject(%#v) error: %v", f.GFWList.Filename, err)
			return ctx, nil, err
		}
		defer resp.Body.Close()

		sites, err := parseAutoProxy(resp.Body)
		if err != nil {
			glog.Errorf("parseAutoProxy(%#v) error: %v", f.GFWList.Filename, err)
			return ctx, nil, err
		}

		sort.Strings(sites)

		io.WriteString(buf, "\nvar sites = {\n")
		for _, site := range sites {
			io.WriteString(buf, "\""+site+"\":1,\n")
		}
		io.WriteString(buf, "\"google.com\":1\n")
		io.WriteString(buf, "}\n")

		io.WriteString(buf, `
function FindProxyForURL(url, host) {
    if ((p = MyFindProxyForURL(url, host)) != "DIRECT") {
        return p
    }

    var lastPos;
    do {
        if (sites.hasOwnProperty(host)) {
            return 'PROXY GOPROXY_ADDRESS';
        }
        lastPos = host.indexOf('.') + 1;
        host = host.slice(lastPos);
    } while (lastPos >= 1);
    return 'DIRECT';
}`)
	}

	s := buf.String()
	f.ProxyPacCache.Set(req.RequestURI, s, time.Now().Add(15*time.Minute))

	s = fixProxyPac(s, req)
	resp = &http.Response{
		StatusCode:    http.StatusOK,
		Header:        http.Header{},
		Request:       req,
		Close:         true,
		ContentLength: int64(len(s)),
		Body:          ioutil.NopCloser(strings.NewReader(s)),
	}

	return ctx, resp, nil
}
コード例 #29
0
ファイル: autoproxy.go プロジェクト: chenzhenjia/goproxy
func (f *Filter) updater() {
	glog.V(2).Infof("start updater for %+v, expiry=%s, duration=%s", f.GFWList.URL.String(), f.GFWList.Expiry, f.GFWList.Duration)

	ticker := time.Tick(f.GFWList.Duration)

	for {
		select {
		case <-ticker:
			glog.V(2).Infof("Begin auto gfwlist(%#v) update...", f.GFWList.URL.String())
			h, err := f.Store.HeadObject(f.GFWList.Filename)
			if err != nil {
				glog.Warningf("stat gfwlist(%#v) err: %v", f.GFWList.Filename, err)
				continue
			}

			lm := h.Get("Last-Modified")
			if lm == "" {
				glog.Warningf("gfwlist(%#v) header(%#v) does not contains last-modified", f.GFWList.Filename, h)
				continue
			}

			modTime, err := time.Parse(f.Store.DateFormat(), lm)
			if err != nil {
				glog.Warningf("stat gfwlist(%#v) has parse %#v error: %v", f.GFWList.Filename, lm, err)
				continue
			}

			if time.Now().Sub(modTime) < f.GFWList.Expiry {
				continue
			}
		}

		glog.Infof("Downloading %#v", f.GFWList.URL.String())

		req, err := http.NewRequest(http.MethodGet, f.GFWList.URL.String(), nil)
		if err != nil {
			glog.Warningf("NewRequest(%#v) error: %v", f.GFWList.URL.String(), err)
			continue
		}

		resp, err := f.Transport.RoundTrip(req)
		if err != nil {
			glog.Warningf("%T.RoundTrip(%#v) error: %v", f.Transport, f.GFWList.URL.String(), err.Error())
			continue
		}

		var r io.Reader = resp.Body
		switch f.GFWList.Encoding {
		case "base64":
			r = base64.NewDecoder(base64.StdEncoding, r)
		default:
			break
		}

		data, err := ioutil.ReadAll(r)
		if err != nil {
			glog.Warningf("ioutil.ReadAll(%T) error: %v", r, err)
			resp.Body.Close()
			continue
		}

		err = f.Store.DeleteObject(f.GFWList.Filename)
		if err != nil {
			glog.Warningf("%T.DeleteObject(%#v) error: %v", f.Store, f.GFWList.Filename, err)
			continue
		}

		err = f.Store.PutObject(f.GFWList.Filename, http.Header{}, ioutil.NopCloser(bytes.NewReader(data)))
		if err != nil {
			glog.Warningf("%T.PutObject(%#v) error: %v", f.Store, f.GFWList.Filename, err)
			continue
		}

		glog.Infof("Update %#v from %#v OK", f.GFWList.Filename, f.GFWList.URL.String())
		resp.Body.Close()
	}
}
コード例 #30
0
ファイル: autorange.go プロジェクト: taiping-z/goproxy
func (f *Filter) Response(ctx context.Context, resp *http.Response) (context.Context, *http.Response, error) {
	if resp.StatusCode != http.StatusPartialContent || resp.Header.Get("Content-Length") == "" {
		return ctx, resp, nil
	}

	if ok1, ok := filters.Bool(ctx, "autorange.default"); ok && ok1 {
		return ctx, resp, nil
	}

	f1 := filters.GetRoundTripFilter(ctx)
	if f1 == nil {
		return ctx, resp, nil
	}
	if _, ok := f.SupportFilters[f1.FilterName()]; !ok {
		glog.V(2).Infof("AUTORANGE hit a unsupported filter=%#v", f1)
		return ctx, resp, nil
	}

	parts := strings.Split(resp.Header.Get("Content-Range"), " ")
	if len(parts) != 2 || parts[0] != "bytes" {
		return ctx, resp, nil
	}

	parts1 := strings.Split(parts[1], "/")
	parts2 := strings.Split(parts1[0], "-")
	if len(parts1) != 2 || len(parts2) != 2 {
		return ctx, resp, nil
	}

	var end, length int64
	var err error

	end, err = strconv.ParseInt(parts2[1], 10, 64)
	if err != nil {
		return ctx, resp, nil
	}

	length, err = strconv.ParseInt(parts1[1], 10, 64)
	if err != nil {
		return ctx, resp, nil
	}

	glog.V(2).Infof("AUTORANGE respone matched, start rangefetch for %#v", resp.Header.Get("Content-Range"))

	resp.ContentLength = length
	resp.Header.Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10))
	resp.Header.Del("Content-Range")

	r, w := AutoPipe(f.Threads)

	go func(w *autoPipeWriter, filter filters.RoundTripFilter, req0 *http.Request, start, length int64) {
		glog.V(2).Infof("AUTORANGE begin rangefetch for %#v by using %#v", req0.URL.String(), filter.FilterName())

		req, err := http.NewRequest(req0.Method, req0.URL.String(), nil)
		if err != nil {
			glog.Warningf("AUTORANGE http.NewRequest(%#v) error: %#v", req, err)
			return
		}

		for key, values := range req0.Header {
			for _, value := range values {
				req.Header.Add(key, value)
			}
		}

		if err := w.WaitForReading(); err != nil {
			return
		}
		var index uint32
		for {
			if w.FatalErr() {
				break
			}
			if start > length-1 {
				// tell reader io.EOF
				w.Close()
				break
			}
			//FIXME: make this configurable!
			if w.Len() > 128*1024*1024 {
				time.Sleep(100 * time.Millisecond)
				continue
			}

			//end := start + int64(f.BufSize-1)
			end := start + int64(1024<<10-1)
			if end > length-1 {
				end = length - 1
			}
			req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))

			_, resp, err := filter.RoundTrip(nil, req)
			if err != nil {
				glog.Warningf("AUTORANGE %#v.RoundTrip(%v) error: %#v", filter, req, err)
				time.Sleep(1 * time.Second)
				continue
			}
			if resp.StatusCode != http.StatusPartialContent {
				if resp.StatusCode >= http.StatusBadRequest {
					time.Sleep(1 * time.Second)
				}
				continue
			}

			w.ThreadHello()
			go func(index uint32, resp *http.Response) {
				defer resp.Body.Close()
				defer w.ThreadBye() // TODO: launch it as soon as iocopy over?

				piper := w.NewPiper(index)
				_, err := helpers.IoCopy(piper, resp.Body)
				if err != nil {
					glog.Warningf("AUTORANGE helpers.IoCopy(%#v) error: %#v", resp.Body, err)
					piper.EIndex()
				}
				piper.WClose()
			}(index, resp)

			start = end + 1
			index++
		}
	}(w, f1, resp.Request, end+1, length)

	resp.Body = helpers.NewMultiReadCloser(resp.Body, r)

	return ctx, resp, nil
}