func fetch(context appengine.Context, ev *event.HTTPRequestEvent) event.Event { errorResponse := new(event.NotifyEvent) errorResponse.SetId(ev.GetId()) req, err := ev.ToRequest("") if nil != err { errorResponse.Code = event.ErrInvalidHttpRequest errorResponse.Reason = fmt.Sprintf("Invalid fetch url:%s with err:%v", ev.URL, err) return errorResponse } var t urlfetch.Transport t.Context = context t.Deadline, _ = time.ParseDuration("10s") t.AllowInvalidServerCertificate = true retryCount := 2 for retryCount > 0 { resp, err := t.RoundTrip(req) if err == nil { res := event.NewHTTPResponseEvent(resp) for nil != resp.Body { buffer := make([]byte, 8192) n, er := resp.Body.Read(buffer) if nil != er { context.Errorf("Failed to read body for reason:%v", er) break } res.Content = append(res.Content, buffer[0:n]...) } if resp.ContentLength != int64(len(res.Content)) { context.Errorf("Failed to read body %d %d", resp.ContentLength, len(res.Content)) } context.Errorf("%v %d %d", resp.Header.Get("Content-Length"), resp.ContentLength, len(res.Content)) return res } context.Errorf("Failed to fetch URL[%s] for reason:%v", ev.URL, err) retryCount-- if strings.EqualFold(req.Method, "GET") && strings.Contains(err.Error(), "RESPONSE_TOO_LARGE") { errorResponse.Code = event.ErrTooLargeResponse return errorResponse } } errorResponse.Code = event.ErrRemoteProxyTimeout errorResponse.Reason = fmt.Sprintf("Fetch timeout for url:%s", ev.URL) return errorResponse }
func (f *RangeFetcher) Fetch(req *event.HTTPRequestEvent) (*event.HTTPResponseEvent, error) { firstRangeStart := int64(0) firstRangeEnd := f.SingleFetchLimit rangeHeader := req.Headers.Get("Range") total := int64(-1) allRangeEnd := int64(-1) if len(rangeHeader) > 0 { fmt.Sscanf(rangeHeader, "bytes=%d-%d", &firstRangeStart, &firstRangeEnd) total = firstRangeEnd - firstRangeStart + 1 allRangeEnd = firstRangeEnd if total > f.SingleFetchLimit { firstRangeEnd = firstRangeStart + f.SingleFetchLimit - 1 } } first, err := rangeFetch(f.C, req, firstRangeStart, firstRangeEnd) if nil != err { return nil, err } if first.StatusCode < 400 { return first, nil } firstChunk := rangeResponseToChunk(first) if nil == firstChunk { return nil, fmt.Errorf("Invalid first range response:%d %v", first.StatusCode, first.Headers) } if total < 0 { total = firstChunk.total allRangeEnd = firstChunk.total - 1 } res := new(event.HTTPResponseEvent) res.SetId(req.GetId()) if len(rangeHeader) > 0 { res.StatusCode = 206 } else { res.StatusCode = 200 } res.Headers = first.Headers res.Headers.Set("Content-Length", fmt.Sprintf("%d", total)) res.Content = first.Content HandleEvent(res) if int64(len(res.Content)) == total { return nil, nil } conccurrentNum := int32(0) nextRangeStart := firstRangeEnd + 1 chunkChannel := make(chan *rangeChunk, f.ConcurrentFetcher) chunkCache := make(map[int64]*rangeChunk) waitNextChunkStart := nextRangeStart stopFetch := false fetch := func(req *event.HTTPRequestEvent, begin, end int64) { defer atomic.AddInt32(&conccurrentNum, -1) res, err := rangeFetch(f.C, req, begin, end) if nil == err { chunk := rangeResponseToChunk(res) if nil != chunk { chunkChannel <- chunk return } } stopFetch = true } for !stopFetch && waitNextChunkStart <= allRangeEnd { for conccurrentNum < f.ConcurrentFetcher && nextRangeStart <= allRangeEnd && int32(len(chunkCache)) < 4*f.ConcurrentFetcher { end := nextRangeStart + f.SingleFetchLimit - 1 if end >= allRangeEnd { end = allRangeEnd } nextRangeStart = allRangeEnd + 1 atomic.AddInt32(&conccurrentNum, 1) go fetch(req, nextRangeStart, end) } select { case chunk := <-chunkChannel: if nil == chunk { stopFetch = true log.Printf("Nil chunk") } else { chunkCache[chunk.start] = chunk for _, chunk := range chunkCache { if waitNextChunkStart == chunk.start { chunkEvent := &event.TCPChunkEvent{} chunkEvent.SetId(req.GetId()) chunkEvent.Content = chunk.content HandleEvent(chunkEvent) waitNextChunkStart = chunk.end + 1 delete(chunkCache, chunk.start) } } } case <-time.After(10 * time.Millisecond): } } close(chunkChannel) log.Printf("Session:%d stop range fetch.", req.GetId()) return nil, nil }
func rangeFetch(C *RemoteChannel, req *event.HTTPRequestEvent, begin, end int64) (*event.HTTPResponseEvent, error) { log.Printf("Session:%d range fetch %d-%d", req.GetId(), begin, end) rangeReq := new(event.HTTPRequestEvent) rangeReq.Headers = make(http.Header) for k, v := range req.Headers { rangeReq.Headers[k] = v } rangeReq.URL = req.URL rangeReq.Method = req.Method rangeReq.SetId(req.GetId()) rangeReq.Headers.Set("Range", fmt.Sprintf("bytes=%d-%d", begin, end)) ev, err := C.Request(rangeReq) if nil == err { res, ok := ev.(*event.HTTPResponseEvent) if ok { res.SetId(req.GetId()) if res.StatusCode == 302 { location := res.Headers.Get("Location") log.Printf("Range fetch:%s redirect to %s", rangeReq.URL, location) rangeReq.URL = location return rangeFetch(C, rangeReq, begin, end) } if res.StatusCode < 400 { return res, nil } } err = fmt.Errorf("Invalid response:%d %v for range fetch", res.StatusCode, res.Headers) } log.Printf("[ERROR]Failed to range fetch[%d-%d] for %s", begin, end, req.URL) return nil, err }