Exemplo n.º 1
0
func fetch(context appengine.Context, ev *event.HTTPRequestEvent) event.Event {
	errorResponse := new(event.NotifyEvent)
	errorResponse.SetId(ev.GetId())
	req, err := ev.ToRequest("")

	if nil != err {
		errorResponse.Code = event.ErrInvalidHttpRequest
		errorResponse.Reason = fmt.Sprintf("Invalid fetch url:%s with err:%v", ev.URL, err)
		return errorResponse
	}
	var t urlfetch.Transport
	t.Context = context
	t.Deadline, _ = time.ParseDuration("10s")
	t.AllowInvalidServerCertificate = true
	retryCount := 2
	for retryCount > 0 {
		resp, err := t.RoundTrip(req)
		if err == nil {
			res := event.NewHTTPResponseEvent(resp)
			for nil != resp.Body {
				buffer := make([]byte, 8192)
				n, er := resp.Body.Read(buffer)
				if nil != er {
					context.Errorf("Failed to read body for reason:%v", er)
					break
				}
				res.Content = append(res.Content, buffer[0:n]...)
			}
			if resp.ContentLength != int64(len(res.Content)) {
				context.Errorf("Failed to read body %d %d", resp.ContentLength, len(res.Content))
			}
			context.Errorf("%v %d %d", resp.Header.Get("Content-Length"), resp.ContentLength, len(res.Content))
			return res
		}
		context.Errorf("Failed to fetch URL[%s] for reason:%v", ev.URL, err)
		retryCount--
		if strings.EqualFold(req.Method, "GET") && strings.Contains(err.Error(), "RESPONSE_TOO_LARGE") {
			errorResponse.Code = event.ErrTooLargeResponse
			return errorResponse
		}
	}
	errorResponse.Code = event.ErrRemoteProxyTimeout
	errorResponse.Reason = fmt.Sprintf("Fetch timeout for url:%s", ev.URL)
	return errorResponse

}
Exemplo n.º 2
0
func (f *RangeFetcher) Fetch(req *event.HTTPRequestEvent) (*event.HTTPResponseEvent, error) {
	firstRangeStart := int64(0)
	firstRangeEnd := f.SingleFetchLimit
	rangeHeader := req.Headers.Get("Range")
	total := int64(-1)
	allRangeEnd := int64(-1)
	if len(rangeHeader) > 0 {
		fmt.Sscanf(rangeHeader, "bytes=%d-%d", &firstRangeStart, &firstRangeEnd)
		total = firstRangeEnd - firstRangeStart + 1
		allRangeEnd = firstRangeEnd
		if total > f.SingleFetchLimit {
			firstRangeEnd = firstRangeStart + f.SingleFetchLimit - 1
		}
	}
	first, err := rangeFetch(f.C, req, firstRangeStart, firstRangeEnd)
	if nil != err {
		return nil, err
	}
	if first.StatusCode < 400 {
		return first, nil
	}
	firstChunk := rangeResponseToChunk(first)
	if nil == firstChunk {
		return nil, fmt.Errorf("Invalid first range response:%d %v", first.StatusCode, first.Headers)
	}
	if total < 0 {
		total = firstChunk.total
		allRangeEnd = firstChunk.total - 1
	}
	res := new(event.HTTPResponseEvent)
	res.SetId(req.GetId())
	if len(rangeHeader) > 0 {
		res.StatusCode = 206
	} else {
		res.StatusCode = 200
	}
	res.Headers = first.Headers
	res.Headers.Set("Content-Length", fmt.Sprintf("%d", total))
	res.Content = first.Content
	HandleEvent(res)
	if int64(len(res.Content)) == total {
		return nil, nil
	}
	conccurrentNum := int32(0)
	nextRangeStart := firstRangeEnd + 1
	chunkChannel := make(chan *rangeChunk, f.ConcurrentFetcher)
	chunkCache := make(map[int64]*rangeChunk)
	waitNextChunkStart := nextRangeStart
	stopFetch := false
	fetch := func(req *event.HTTPRequestEvent, begin, end int64) {
		defer atomic.AddInt32(&conccurrentNum, -1)
		res, err := rangeFetch(f.C, req, begin, end)
		if nil == err {
			chunk := rangeResponseToChunk(res)
			if nil != chunk {
				chunkChannel <- chunk
				return
			}
		}
		stopFetch = true
	}

	for !stopFetch && waitNextChunkStart <= allRangeEnd {
		for conccurrentNum < f.ConcurrentFetcher && nextRangeStart <= allRangeEnd && int32(len(chunkCache)) < 4*f.ConcurrentFetcher {
			end := nextRangeStart + f.SingleFetchLimit - 1
			if end >= allRangeEnd {
				end = allRangeEnd
			}
			nextRangeStart = allRangeEnd + 1
			atomic.AddInt32(&conccurrentNum, 1)
			go fetch(req, nextRangeStart, end)
		}
		select {
		case chunk := <-chunkChannel:
			if nil == chunk {
				stopFetch = true
				log.Printf("Nil chunk")
			} else {
				chunkCache[chunk.start] = chunk
				for _, chunk := range chunkCache {
					if waitNextChunkStart == chunk.start {
						chunkEvent := &event.TCPChunkEvent{}
						chunkEvent.SetId(req.GetId())
						chunkEvent.Content = chunk.content
						HandleEvent(chunkEvent)
						waitNextChunkStart = chunk.end + 1
						delete(chunkCache, chunk.start)
					}
				}
			}
		case <-time.After(10 * time.Millisecond):
		}
	}
	close(chunkChannel)
	log.Printf("Session:%d stop range fetch.", req.GetId())
	return nil, nil
}
Exemplo n.º 3
0
func rangeFetch(C *RemoteChannel, req *event.HTTPRequestEvent, begin, end int64) (*event.HTTPResponseEvent, error) {
	log.Printf("Session:%d range fetch %d-%d", req.GetId(), begin, end)
	rangeReq := new(event.HTTPRequestEvent)
	rangeReq.Headers = make(http.Header)
	for k, v := range req.Headers {
		rangeReq.Headers[k] = v
	}
	rangeReq.URL = req.URL
	rangeReq.Method = req.Method
	rangeReq.SetId(req.GetId())
	rangeReq.Headers.Set("Range", fmt.Sprintf("bytes=%d-%d", begin, end))
	ev, err := C.Request(rangeReq)
	if nil == err {
		res, ok := ev.(*event.HTTPResponseEvent)
		if ok {
			res.SetId(req.GetId())
			if res.StatusCode == 302 {
				location := res.Headers.Get("Location")
				log.Printf("Range fetch:%s redirect to %s", rangeReq.URL, location)
				rangeReq.URL = location
				return rangeFetch(C, rangeReq, begin, end)
			}
			if res.StatusCode < 400 {
				return res, nil
			}
		}
		err = fmt.Errorf("Invalid response:%d %v for range fetch", res.StatusCode, res.Headers)
	}
	log.Printf("[ERROR]Failed to range fetch[%d-%d] for %s", begin, end, req.URL)
	return nil, err
}