// Use a custom http client rather than http.DefaultClient. We limit ourselves to a maximum of |requestLimit| concurrent http requests, the custom httpClient ups the maxIdleConnsPerHost value so that one connection stays open for each concurrent request.
func makeHTTPClient(requestLimit int) *http.Client {
t := http.Transport(*http.DefaultTransport.(*http.Transport))
t.MaxIdleConnsPerHost = requestLimit
// This sets, essentially, an idle-timeout. The timer starts counting AFTER the client has finished sending the entire request to the server. As soon as the client receives the server's response headers, the timeout is canceled.
t.ResponseHeaderTimeout = time.Duration(2) * time.Minute
return &http.Client{Transport: &t}
}
// Use a custom http client rather than http.DefaultClient. We limit ourselves to a maximum of |requestLimit| concurrent http requests, the custom httpClient ups the maxIdleConnsPerHost value so that one connection stays open for each concurrent request.
func makeHTTPClient() *http.Client {
t := http.Transport(*http.DefaultTransport.(*http.Transport))
t.MaxIdleConnsPerHost = requestLimit
return &http.Client{
Transport: &t,
Timeout: time.Duration(30) * time.Second,
}
}