Exemplo n.º 1
0
func (gae *GAEHttpConnection) doRangeFetch(req *http.Request, firstChunkRes *http.Response) {
	task := new(rangeFetchTask)
	task.FetchLimit = int(gae_cfg.FetchLimitSize)
	task.FetchWorkerNum = int(gae_cfg.ConcurrentRangeFetcher)
	task.SessionID = gae.sess.SessionID
	//	task.TaskValidation = func() bool {
	//		return !util.IsDeadConnection(gae.sess.LocalRawConn)
	//	}
	gae.rangeWorker = task
	fetch := func(preq *http.Request) (*http.Response, error) {
		ev := new(event.HTTPRequestEvent)
		ev.FromRequest(preq)
		ev.SetHash(gae.sess.SessionID)
		err, xres := gae.requestEvent(gaeHttpClient, gae.sess, ev)
		if nil != err {
			//try again
			err, xres = gae.requestEvent(gaeHttpClient, gae.sess, ev)
		}
		if nil == err {
			httpresev := xres.(*event.HTTPResponseEvent)
			httpres := httpresev.ToResponse()
			return httpres, err
		}
		return nil, err
	}
	pres, err := task.SyncGet(req, firstChunkRes, fetch)
	if nil == err {
		err = pres.Write(gae.sess.LocalRawConn)
		if nil != err {
			task.Close()
			gae.rangeWorker = nil
		}
		if nil != pres.Body {
			pres.Body.Close()
		}
	}
	if nil != err {
		log.Printf("Session[%d]Range task failed for reason:%v\n", gae.sess.SessionID, err)
	}
	if nil != err || !util.IsResponseKeepAlive(pres) || !util.IsRequestKeepAlive(req) {
		gae.sess.LocalRawConn.Close()
		gae.sess.State = STATE_SESSION_CLOSE
		gae.Close()
	}
}
Exemplo n.º 2
0
func (session *SessionConnection) processHttpEvent(ev *event.HTTPRequestEvent) error {
	ev.SetHash(session.SessionID)
	//proxies, attrs := SelectProxy(ev.RawReq, session.LocalRawConn, session.Type == HTTPS_TUNNEL)
	proxies, attrs := SelectProxy(ev.RawReq, session)
	if nil == proxies {
		session.State = STATE_SESSION_CLOSE
		return nil
	}
	var err error
	if nil == session.RemoteConn {
		err = session.tryProxy(proxies, attrs, ev)
	} else {
		rmanager := session.RemoteConn.GetConnectionManager()
		matched := false
		for _, proxy := range proxies {
			proxyName := adjustProxyName(proxy.GetName(), session.Type == HTTPS_TUNNEL)
			if rmanager.GetName() == proxyName {
				matched = true
				break
			}
		}
		if !matched {
			session.RemoteConn.Close()
			err = session.tryProxy(proxies, attrs, ev)
		} else {
			err, _ = session.RemoteConn.Request(session, ev)
		}
	}

	if nil != err {
		log.Printf("Session[%d]Process error:%v for host:%s", session.SessionID, err, ev.RawReq.Host)
		session.LocalRawConn.Write([]byte("HTTP/1.1 500 Internal Server Error\r\n\r\n"))
		session.LocalRawConn.Close()
	}
	return nil
}
Exemplo n.º 3
0
func (c4 *C4RemoteSession) writeHttpRequest(preq *http.Request) error {
	ev := new(event.HTTPRequestEvent)
	ev.FromRequest(preq)
	ev.SetHash(c4.sess.SessionID)
	if strings.Contains(ev.Url, "http://") {
		ev.Url = ev.Url[7+len(preq.Host):]
	}
	//log.Printf("Session[%d]Range Request %s\n", c4.sess.SessionID, ev.Url)
	c4.offerRequestEvent(ev)
	return nil
}
Exemplo n.º 4
0
func Fetch(context appengine.Context, ev *event.HTTPRequestEvent) event.Event {
	errorResponse := new(event.HTTPResponseEvent)
	if Cfg.IsMaster == 1 {
		fillErrorResponse(errorResponse, "Proxy service is no enable in snova master node.")
		return errorResponse
	}
	if isInBlacklist(ev.GetHeader("Host")) {
		fillErrorResponse(errorResponse, "Current site is in blacklist.")
		return errorResponse
	}
	req := buildHTTPRequest(ev)

	if req == nil {
		errorResponse.Status = 400
		fillErrorResponse(errorResponse, "Invalid fetch url:"+ev.Url)
		return errorResponse
	}
	var t urlfetch.Transport
	t.Context = context
	t.Deadline, _ = time.ParseDuration("10s")
	t.AllowInvalidServerCertificate = true
	//t := &transport
	//t := &urlfetch.Transport{context, 0, true}
	retryCount := Cfg.RetryFetchCount
	for retryCount > 0 {
		resp, err := t.RoundTrip(req)
		if err == nil {
			res := buildHTTPResponseEvent(resp)
			if res.Status == 302 {
				rangeHeader := req.Header.Get("Range")
				if len(rangeHeader) > 0 {
					res.AddHeader("X-Range", rangeHeader)
				}
			}
			return res
		}
		context.Errorf("Failed to fetch URL[%s] for reason:%v", ev.Url, err)
		retryCount--
		if strings.EqualFold(req.Method, "GET") && strings.Contains(err.Error(), "RESPONSE_TOO_LARGE") {
			rangeLimit := Cfg.RangeFetchLimit
			rangestart := 0
			rangeheader := req.Header.Get("Range")
			if len(rangeheader) > 0 {
				rangestart, _ = util.ParseRangeHeaderValue(rangeheader)
			}
			req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", rangestart, rangeLimit-1))
		}
		if strings.Contains(err.Error(), "RESPONSE_TOO_LARGE") {
			time.Sleep(1 * time.Second)
			return Fetch(context, ev)
		}

	}
	errorResponse.Status = 408
	fillErrorResponse(errorResponse, "Fetch timeout for url:"+ev.Url)
	rangeHeader := req.Header.Get("Range")
	if len(rangeHeader) > 0 {
		errorResponse.SetHeader("X-Range", rangeHeader)
	}
	return errorResponse

}
Exemplo n.º 5
0
func (session *SessionConnection) process() error {
	close_session := func() {
		session.LocalRawConn.Close()
		if nil != session.RemoteConn {
			session.RemoteConn.Close()
		}
		session.State = STATE_SESSION_CLOSE
	}

	readRequest := func() (*http.Request, error) {
		for {
			needCheckRemote := false
			if nil != session.RemoteConn {
				forward, ok := session.RemoteConn.(*ForwardConnection)
				if ok {
					if forward.IsClosed() {
						return nil, fmt.Errorf("Remote conn closed.")
					}
					needCheckRemote = true
				}
			}
			if needCheckRemote {
				session.LocalRawConn.SetReadDeadline(time.Now().Add(1 * time.Second))
			}
			req, e := http.ReadRequest(session.LocalBufferConn)
			if nil != req {
				req.Header.Del("Proxy-Connection")
			}
			if needCheckRemote {
				var zero time.Time
				session.LocalRawConn.SetReadDeadline(zero)
			}
			if nil != e && util.IsTimeoutError(e) {
				continue
			}
			return req, e
		}
		return nil, nil
	}

	switch session.State {
	case STATE_RECV_HTTP:
		req, rerr := readRequest()
		if nil == rerr {
			var rev event.HTTPRequestEvent
			rev.FromRequest(req)
			rev.SetHash(session.SessionID)
			err := session.processHttpEvent(&rev)
			if err != nil {
				log.Printf("Session[%d]Failed to read http request:%v\n", session.SessionID, err)
				close_session()
				return io.EOF
			}
		}
		if nil != rerr {
			log.Printf("Session[%d]Browser close connection:%v\n", session.SessionID, rerr)
			close_session()
			return io.EOF
		}
	case STATE_RECV_HTTP_CHUNK:
		buf := make([]byte, 8192)
		n, err := session.LocalBufferConn.Read(buf)
		if nil == err {
			rev := new(event.HTTPChunkEvent)
			rev.Content = buf[0:n]
			err = session.processHttpChunkEvent(rev)
		}
		if nil != err {
			close_session()

			return io.EOF
		}
	case STATE_RECV_TCP:

	}
	return nil
}