func (hf HttpFetcher) GetAll(urls []string) ([]*HttpFetcherResponse, error) {
m := metrics.GetOrRegisterTimer("fn.FetchRemoteData", nil) // TODO: update metric name
defer m.UpdateSince(time.Now())
resps := make([]*HttpFetcherResponse, len(urls))
var wg sync.WaitGroup
wg.Add(len(urls))
// TODO: add thruput here..
for i, urlStr := range urls {
resps[i] = &HttpFetcherResponse{}
go func(resp *HttpFetcherResponse) {
defer wg.Done()
url, err := urlx.Parse(urlStr)
if err != nil {
resp.Err = err
return
}
resp.URL = url
lg.Infof("Fetching %s", url.String())
fetch, err := hf.client().Get(url.String())
if err != nil {
lg.Warnf("Error fetching %s because %s", url.String(), err)
resp.Err = err
return
}
defer fetch.Body.Close()
resp.Status = fetch.StatusCode
body, err := ioutil.ReadAll(fetch.Body)
if err != nil {
resp.Err = err
return
}
resp.Data = body
resp.Err = nil
}(resps[i])
}
wg.Wait()
return resps, nil
}
func (f Fetcher) GetAll(ctx context.Context, urls []string) ([]*FetcherResponse, error) {
defer metrics.MeasureSince([]string{"fn.FetchRemoteData"}, time.Now())
fetches := make([]*FetcherResponse, len(urls))
var wg sync.WaitGroup
wg.Add(len(urls))
// TODO: add thruput here..
for i, urlStr := range urls {
fetches[i] = &FetcherResponse{}
go func(fetch *FetcherResponse) {
defer wg.Done()
url, err := urlx.Parse(urlStr)
if err != nil {
fetch.Err = err
return
}
fetch.URL = url
lg.Infof("Fetching %s", url.String())
resp, err := ctxhttp.Get(ctx, f.client(), url.String())
if err != nil {
lg.Warnf("Error fetching %s because %s", url.String(), err)
fetch.Err = err
return
}
defer resp.Body.Close()
fetch.Status = resp.StatusCode
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
fetch.Err = err
return
}
fetch.Data = body
fetch.Err = nil
}(fetches[i])
}
wg.Wait()
return fetches, nil
}
