func (hf HttpFetcher) GetAll(urls []string) ([]*HttpFetcherResponse, error) { m := metrics.GetOrRegisterTimer("fn.FetchRemoteData", nil) // TODO: update metric name defer m.UpdateSince(time.Now()) resps := make([]*HttpFetcherResponse, len(urls)) var wg sync.WaitGroup wg.Add(len(urls)) // TODO: add thruput here.. for i, urlStr := range urls { resps[i] = &HttpFetcherResponse{} go func(resp *HttpFetcherResponse) { defer wg.Done() url, err := urlx.Parse(urlStr) if err != nil { resp.Err = err return } resp.URL = url lg.Infof("Fetching %s", url.String()) fetch, err := hf.client().Get(url.String()) if err != nil { lg.Warnf("Error fetching %s because %s", url.String(), err) resp.Err = err return } defer fetch.Body.Close() resp.Status = fetch.StatusCode body, err := ioutil.ReadAll(fetch.Body) if err != nil { resp.Err = err return } resp.Data = body resp.Err = nil }(resps[i]) } wg.Wait() return resps, nil }
func (f Fetcher) GetAll(ctx context.Context, urls []string) ([]*FetcherResponse, error) { defer metrics.MeasureSince([]string{"fn.FetchRemoteData"}, time.Now()) fetches := make([]*FetcherResponse, len(urls)) var wg sync.WaitGroup wg.Add(len(urls)) // TODO: add thruput here.. for i, urlStr := range urls { fetches[i] = &FetcherResponse{} go func(fetch *FetcherResponse) { defer wg.Done() url, err := urlx.Parse(urlStr) if err != nil { fetch.Err = err return } fetch.URL = url lg.Infof("Fetching %s", url.String()) resp, err := ctxhttp.Get(ctx, f.client(), url.String()) if err != nil { lg.Warnf("Error fetching %s because %s", url.String(), err) fetch.Err = err return } defer resp.Body.Close() fetch.Status = resp.StatusCode body, err := ioutil.ReadAll(resp.Body) if err != nil { fetch.Err = err return } fetch.Data = body fetch.Err = nil }(fetches[i]) } wg.Wait() return fetches, nil }