Beispiel #1
0
func (hf HttpFetcher) GetAll(urls []string) ([]*HttpFetcherResponse, error) {
	m := metrics.GetOrRegisterTimer("fn.FetchRemoteData", nil) // TODO: update metric name
	defer m.UpdateSince(time.Now())

	resps := make([]*HttpFetcherResponse, len(urls))

	var wg sync.WaitGroup
	wg.Add(len(urls))

	// TODO: add thruput here..

	for i, urlStr := range urls {
		resps[i] = &HttpFetcherResponse{}

		go func(resp *HttpFetcherResponse) {
			defer wg.Done()

			url, err := urlx.Parse(urlStr)
			if err != nil {
				resp.Err = err
				return
			}
			resp.URL = url

			lg.Infof("Fetching %s", url.String())

			fetch, err := hf.client().Get(url.String())
			if err != nil {
				lg.Warnf("Error fetching %s because %s", url.String(), err)
				resp.Err = err
				return
			}
			defer fetch.Body.Close()

			resp.Status = fetch.StatusCode

			body, err := ioutil.ReadAll(fetch.Body)
			if err != nil {
				resp.Err = err
				return
			}
			resp.Data = body
			resp.Err = nil

		}(resps[i])
	}

	wg.Wait()
	return resps, nil
}
Beispiel #2
0
func (f Fetcher) GetAll(ctx context.Context, urls []string) ([]*FetcherResponse, error) {
	defer metrics.MeasureSince([]string{"fn.FetchRemoteData"}, time.Now())

	fetches := make([]*FetcherResponse, len(urls))

	var wg sync.WaitGroup
	wg.Add(len(urls))

	// TODO: add thruput here..

	for i, urlStr := range urls {
		fetches[i] = &FetcherResponse{}

		go func(fetch *FetcherResponse) {
			defer wg.Done()

			url, err := urlx.Parse(urlStr)
			if err != nil {
				fetch.Err = err
				return
			}
			fetch.URL = url

			lg.Infof("Fetching %s", url.String())

			resp, err := ctxhttp.Get(ctx, f.client(), url.String())
			if err != nil {
				lg.Warnf("Error fetching %s because %s", url.String(), err)
				fetch.Err = err
				return
			}
			defer resp.Body.Close()

			fetch.Status = resp.StatusCode

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				fetch.Err = err
				return
			}
			fetch.Data = body
			fetch.Err = nil

		}(fetches[i])
	}

	wg.Wait()
	return fetches, nil
}