Example #1
0
//provides a httpResponse for a GET,DELETE,POST or PUT. Can support json data, use "json" as a key on the parmeter
func Curl(p RequestParms) (*http.Response, error) {
	var client http.Client
	var req *http.Request
	var resp *http.Response
	var err error

	client.Timeout = p.Timeout
	if client.Timeout == 0 {
		client.Timeout = time.Second * DEFAULT_TIMEOUT
	}

	if p.Method == HTTP_GET || p.Method == HTTP_DELETE {
		url := buildGetUrl(p.Params, p.Endpoint)
		req, _ = http.NewRequest(p.Method, url, nil)
		p.populateHeaders(req)
		resp, err = client.Do(req)
	}
	if p.Method == HTTP_POST || p.Method == HTTP_PUT {
		url := p.Endpoint
		data := formValues(p.Params)
		req, err = http.NewRequest(p.Method, url, bytes.NewBufferString(data.Encode()))
		if err != nil {
			log.Println(err)
		}

		req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
		if p.Headers == nil {
			req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
		} else {
			p.populateHeaders(req)
		}

		resp, err = client.Do(req)
	}
	if p.Method == HTTP_JSONPOST {
		url := p.Endpoint
		data := p.Params["json"]
		req, err = http.NewRequest("POST", url, bytes.NewBufferString(data))
		if err != nil {
			log.Println(err)
		}
		req.Header.Set("Content-Type", "application/json")
		req.Header.Add("Content-Length", strconv.Itoa(len(data)))
		p.populateHeaders(req)
		resp, err = client.Do(req)
	}
	return resp, err
}
Example #2
0
// startupHealthcheck is used at startup to check if the server is available
// at all.
func (c *Client) startupHealthcheck(timeout time.Duration) error {
	c.mu.Lock()
	urls := c.urls
	c.mu.Unlock()

	// If we don't get a connection after "timeout", we bail.
	start := time.Now()
	for {
		// Make a copy of the HTTP client provided via options to respect
		// settings like Basic Auth or a user-specified http.Transport.
		cl := new(http.Client)
		*cl = *c.c
		cl.Timeout = timeout

		for _, url := range urls {
			res, err := cl.Head(url)
			if err == nil && res != nil && res.StatusCode >= 200 && res.StatusCode < 300 {
				return nil
			}
		}
		time.Sleep(1 * time.Second)
		if time.Now().Sub(start) > timeout {
			break
		}
	}
	return ErrNoClient
}
func main() {
	runtime.GOMAXPROCS(4)
	scanner := bufio.NewScanner(os.Stdin)
	hosts := make(chan string)
	client := http.Client{}
	client.Timeout = 2 * time.Second
	seen := make(map[string]struct{})
	wg := sync.WaitGroup{}
	go func() {
		for host := range hosts {
			fmt.Println(host)
		}
	}()
	for scanner.Scan() {
		targetUrl := scanner.Text()
		if _, there := seen[targetUrl]; !there {
			wg.Add(1)
			go func() {
				defer wg.Done()
				resolve(targetUrl, client, hosts)
			}()
			seen[targetUrl] = struct{}{}
		}
	}
	wg.Wait()
}
Example #4
0
// Also you can use;
// bot.whatismyipaddress.com
// https://api.ipify.org
func GetWANIP() (string, error) {

	var hc http.Client
	var reader io.Reader

	req, err := http.NewRequest("GET", "http://icanhazip.com", reader)
	if err != nil {
		return "", err
	}

	hc.Timeout = time.Duration(4) * time.Second
	resp, err := hc.Do(req)

	if err != nil {
		return "", err
	}

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return "", err
	}

	defer resp.Body.Close()

	ipstr := strings.TrimSpace(string(body))

	if !isValidIPv4(ipstr) {
		return "", errors.New("Invalid IPv4 address: " + ipstr)
	}

	return ipstr, err
}
Example #5
0
func TestRuntime_OverrideClientOperation(t *testing.T) {
	client := &http.Client{}
	rt := NewWithClient("", "/", []string{"https"}, client)
	var i int
	rt.clientOnce.Do(func() { i++ })
	assert.Equal(t, client, rt.client)
	assert.Equal(t, 0, i)

	var seen *http.Client
	rt.do = func(_ context.Context, cl *http.Client, _ *http.Request) (*http.Response, error) {
		seen = cl
		res := new(http.Response)
		res.StatusCode = 200
		res.Body = ioutil.NopCloser(bytes.NewBufferString("OK"))
		return res, nil
	}

	client2 := new(http.Client)
	client2.Timeout = 3 * time.Second
	if assert.NotEqual(t, client, client2) {
		_, err := rt.Submit(&runtime.ClientOperation{
			Client: client2,
			Params: runtime.ClientRequestWriterFunc(func(r runtime.ClientRequest, _ strfmt.Registry) error {
				return nil
			}),
			Reader: runtime.ClientResponseReaderFunc(func(_ runtime.ClientResponse, _ runtime.Consumer) (interface{}, error) {
				return nil, nil
			}),
		})
		if assert.NoError(t, err) {

			assert.Equal(t, client2, seen)
		}
	}
}
Example #6
0
// UnversionedRESTClientFor is the same as RESTClientFor, except that it allows
// the config.Version to be empty.
func UnversionedRESTClientFor(config *Config) (*RESTClient, error) {
	if config.NegotiatedSerializer == nil {
		return nil, fmt.Errorf("NeogitatedSerializer is required when initializing a RESTClient")
	}

	baseURL, versionedAPIPath, err := defaultServerUrlFor(config)
	if err != nil {
		return nil, err
	}

	transport, err := TransportFor(config)
	if err != nil {
		return nil, err
	}

	var httpClient *http.Client
	if transport != http.DefaultTransport {
		httpClient = &http.Client{Transport: transport}
		if config.Timeout > 0 {
			httpClient.Timeout = config.Timeout
		}
	}

	versionConfig := config.ContentConfig
	if versionConfig.GroupVersion == nil {
		v := metav1.SchemeGroupVersion
		versionConfig.GroupVersion = &v
	}

	return NewRESTClient(baseURL, versionedAPIPath, versionConfig, config.QPS, config.Burst, config.RateLimiter, httpClient)
}
Example #7
0
func get(url_ string, header http.Header, timeout time.Duration) (*http.Response, error) {
	for i := 0; i < 10; i += 1 {
		client := http.Client{CheckRedirect: CheckRedirect}
		client.Timeout = timeout
		req, err := http.NewRequest("GET", url_, nil)
		if err != nil {
			return nil, err
		}
		// log.Printf("url: %v\n\n", req.URL)
		req.Header = header
		resp, err := client.Do(req)
		// log.Print(resp.Header)
		if err != nil {
			switch err.(*url.Error).Err.(type) {
			case RedirectError:
				tmp, err := resp.Location()
				if err != nil {
					return nil, err
				}
				url_ = tmp.String()
				continue
			default:
				return nil, err
			}
		} else {
			return resp, err
		}
	}
	return nil, fmt.Errorf("unknow error")
}
Example #8
0
// submitRequest uses a given client and submits the specified request.
func (c *SecureContext) submitRequest(rw http.ResponseWriter, req *http.Request, url string, client *http.Client, responseHandler ResponseHandler) {
	// Prevents lingering goroutines from living forever.
	// http://stackoverflow.com/questions/16895294/how-to-set-timeout-for-http-get-requests-in-golang/25344458#25344458
	client.Timeout = 5 * time.Second
	// In case the body is not of io.Closer.
	if req.Body != nil {
		defer req.Body.Close()
	}
	req.Close = true
	// Make a new request.
	request, _ := http.NewRequest(req.Method, url, req.Body)
	// In case the body is not of io.Closer.
	if request.Body != nil {
		defer request.Body.Close()
	}
	request.Close = true
	// Send the request.
	res, err := client.Do(request)
	if res != nil {
		defer res.Body.Close()
	}
	if err != nil {
		rw.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(rw, "unknown error. try again")
		return
	}
	// Should return the same status.
	rw.WriteHeader(res.StatusCode)
	responseHandler(&rw, res)
}
Example #9
0
// Test a HTTP call.
func testHttp(test HttpHealthCheckAttributes, handler ResponseHandler) {
	client := http.Client{}
	client.Timeout = test.Timeout
	request := &http.Request{
		Method: "GET",
		URL:    test.Url,
		Close:  true,
	}
	time_start := time.Now()
	res, err := client.Do(request)
	total_time := time.Since(time_start)

	if err != nil {
		handler.Failed(map[string]interface{}{
			"error": fmt.Errorf("HTTP call failed with: %s", err),
		})
		return
	}

	res.Body.Close()

	if res.StatusCode == http.StatusOK {
		handler.Passed(map[string]interface{}{
			"latency": total_time,
		})
		return
	}

	handler.Failed(map[string]interface{}{
		"error": fmt.Errorf("Invalid response code: %s", res.StatusCode),
	})
}
Example #10
0
// NewSender creates a new Sender and sets a timeout on the http.Client
func NewSender(apiKey string, retryCount int, timeout time.Duration) *Client {
	httpClient := new(http.Client)
	httpClient.Timeout = timeout
	return &Client{
		APIKey:     apiKey,
		RetryCount: retryCount,
		HTTPClient: httpClient,
	}
}
Example #11
0
func (c Client) exec(verb, url string, headers map[string]string, body io.Reader) (*storageResponse, error) {
	authHeader, err := c.getAuthorizationHeader(verb, url, headers)
	if err != nil {
		return nil, err
	}
	headers["Authorization"] = authHeader

	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest(verb, url, body)
	for k, v := range headers {
		req.Header.Add(k, v)
	}
	httpClient := http.Client{}

	httpClient.Timeout = c.Timeout

	resp, err := httpClient.Do(req)
	if err != nil {
		return nil, err
	}

	statusCode := resp.StatusCode
	if statusCode >= 400 && statusCode <= 505 {
		var respBody []byte
		respBody, err = readResponseBody(resp)
		if err != nil {
			return nil, err
		}

		if len(respBody) == 0 {
			// no error in response body
			err = fmt.Errorf("storage: service returned without a response body (%s)", resp.Status)
		} else {
			// response contains storage service error object, unmarshal
			storageErr, errIn := serviceErrFromXML(respBody, resp.StatusCode, resp.Header.Get("x-ms-request-id"))
			if err != nil { // error unmarshaling the error response
				err = errIn
			}
			err = storageErr
		}
		return &storageResponse{
			statusCode: resp.StatusCode,
			headers:    resp.Header,
			body:       ioutil.NopCloser(bytes.NewReader(respBody)), /* restore the body */
		}, err
	}

	return &storageResponse{
		statusCode: resp.StatusCode,
		headers:    resp.Header,
		body:       resp.Body}, nil
}
Example #12
0
// Creates a new Trade based on the given cookies `sessionid`, `steamLogin`, `steamLoginSecure` and the trade partner's Steam ID.
func New(sessionId, steamLogin, steamLoginSecure string, other steamid.SteamId) *Trade {
	client := new(http.Client)
	client.Timeout = 10 * time.Second

	t := &Trade{
		client:    client,
		other:     other,
		sessionId: sessionId,
		baseUrl:   fmt.Sprintf(tradeUrl, other),
		Version:   1,
	}
	community.SetCookies(t.client, sessionId, steamLogin, steamLoginSecure)
	return t
}
Example #13
0
func (br *Bracelet) GetImage(str string) (string, error) {
	client := http.Client{}
	client.Timeout = time.Second * 10

	v := url.Values{}
	v.Set("str", str)

	resp, err := client.PostForm("http://"+br.Address.ImConv+"/", v)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	buf := &bytes.Buffer{}
	buf.ReadFrom(resp.Body)

	return hex.EncodeToString(buf.Bytes()), nil
}
Example #14
0
// LookupIPWithClient looks up the given IP using a geolocation service and returns a
// City struct. If an httpClient was provided, it uses that, otherwise it uses
// a default http.Client.
func LookupIPWithClient(ipAddr string, httpClient *http.Client) (*City, string, error) {
	if httpClient == nil {
		log.Trace("Using default http.Client")
		httpClient = defaultHttpClient
	}
	httpClient.Timeout = geoLookupTimeout

	var err error
	var req *http.Request
	var resp *http.Response

	lookupURL := fmt.Sprintf(geoServeEndpoint, ipAddr)

	if req, err = http.NewRequest("GET", lookupURL, nil); err != nil {
		return nil, "", fmt.Errorf("Could not create request: %q", err)
	}

	if resp, err = httpClient.Do(req); err != nil {
		return nil, "", fmt.Errorf("Could not get response from server: %q", err)
	}
	defer func() {
		if err := resp.Body.Close(); err != nil {
			log.Debugf("Unable to close reponse body: %v", err)
		}
	}()

	if resp.StatusCode != http.StatusOK {
		body := "body unreadable"
		b, err := ioutil.ReadAll(resp.Body)
		if err == nil {
			body = string(b)
		}
		return nil, "", fmt.Errorf("Unexpected response status %d: %v", resp.StatusCode, body)
	}

	ip := resp.Header.Get("X-Reflected-Ip")

	decoder := json.NewDecoder(resp.Body)

	city := &City{}
	if err = decoder.Decode(city); err != nil {
		return nil, ip, err
	}

	return city, ip, nil
}
Example #15
0
File: main.go Project: khaf/rest
func (self *Client) do(req *http.Request) (*http.Response, error) {
	client := new(http.Client)

	// Adding cookie jar
	if self.CookieJar != nil {
		client.Jar = self.CookieJar
	}

	// Copying headers
	for k := range self.Header {
		req.Header.Set(k, self.Header.Get(k))
	}

	if req.Body == nil {
		req.Header.Del("Content-Type")
		req.Header.Del("Content-Length")
	}

	client.Timeout = self.Timeout

	res, err := client.Do(req)

	if debugLevelEnabled(debugLevelVerbose) {

		log.Printf("Fetching %v\n", req.URL.String())

		log.Printf("> %s %s", req.Method, req.Proto)
		for k := range req.Header {
			for kk := range req.Header[k] {
				log.Printf("> %s: %s", k, req.Header[k][kk])
			}
		}

		log.Printf("< %s %s", res.Proto, res.Status)
		for k := range res.Header {
			for kk := range res.Header[k] {
				log.Printf("< %s: %s", k, res.Header[k][kk])
			}
		}

		log.Printf("\n")
	}

	return res, err
}
Example #16
0
// Do creates an http.Client and runs Do
func (doer *HTTPDoer) Do(timeout time.Duration) error {
	client := new(http.Client)
	client.Timeout = timeout

	response, err := client.Do(doer.request)
	doer.response = response
	if err != nil {
		return err
	}
	defer doer.response.Body.Close()

	doer.responseBytes, err = ioutil.ReadAll(doer.response.Body)
	if err != nil {
		return err
	}

	return nil
}
Example #17
0
//execute download from given source
func downloadWithTransport(httpClient *http.Client, url string, param *Param) (*Result, error) {
	result := new(Result)

	//test if host-port accept connections
	isPortOpen, err := TestOpenTcpPort(param)
	if !isPortOpen || err != nil {
		return result, err
	}
	result.PortOpen = true

	//set timeout
	httpClient.Timeout = param.DownloadTimeout

	//get request
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return result, err
	}
	req.Close = true

	//add custom headers
	req.Header.Add("User-Agent", param.UserAgent)

	//let's try to fetch data
	resp, err := httpClient.Do(req)
	if err != nil {
		return result, err
	}
	defer resp.Body.Close()

	//define result
	result.ResponseCode = resp.StatusCode

	//try to get the body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return result, err
	}
	result.Body = string(body)
	result.Ok = true

	return result, nil
}
Example #18
0
func main() {
	c := new(http.Client)
	req := request.NewRequest(c)

	fmt.Println("default timeout")
	diff(req)

	timeout := time.Duration(1 * time.Second)
	c.Timeout = timeout
	fmt.Printf("set timeout = %f seconds\n", timeout.Seconds())
	diff(req)

	// Or use req.Client
	c = new(http.Client)
	req = request.NewRequest(c)
	req.Client.Timeout = timeout
	fmt.Printf("set timeout = %f seconds\n", timeout.Seconds())
	diff(req)
}
Example #19
0
func SendHttpRequest(cache *HtmlCache, httpTimeout time.Duration) (resp *http.Response, err error) {
	if *gVerbose {
		log.Printf("start to request %s", cache.URL)
	}

	req, err := http.NewRequest("GET", cache.URL.String(), nil)
	if nil != err {
		log.Printf("[ERROR] failed to create http request for %s: %s", cache.URL, err)
		return
	}

	req.Header.Set("User-Agent", GOFEED_AGENT)
	// set cache related headers
	if "" != cache.CacheControl {
		req.Header.Set("Cache-Control", cache.CacheControl)
	}
	if nil != cache.LastModified {
		req.Header.Set("If-Modified-Since", cache.LastModified.Format(http.TimeFormat))
	}
	if "" != cache.Etag {
		req.Header.Set("If-None-Match", cache.Etag)
	}

	// set cache date as request date
	dateNow := time.Now()
	cache.Date = &dateNow

	// send request
	client := new(http.Client)
	client.Timeout = httpTimeout
	resp, err = client.Do(req)
	if nil != err {
		log.Printf("[ERROR] http client failed to send request to %s: %s", cache.URL.String(), err)
		return
	}

	return
}
Example #20
0
func lookupIp(httpClient *http.Client) (string, string, error) {
	httpClient.Timeout = 60 * time.Second

	var err error
	var req *http.Request
	var resp *http.Response

	// Note this will typically be an HTTP client that uses direct domain fronting to
	// hit our server pool in the Netherlands.
	if req, err = http.NewRequest("HEAD", "http://nl.fallbacks.getiantem.org", nil); err != nil {
		return "", "", fmt.Errorf("Could not create request: %q", err)
	}

	if resp, err = httpClient.Do(req); err != nil {
		return "", "", fmt.Errorf("Could not get response from server: %q", err)
	}
	defer func() {
		if err := resp.Body.Close(); err != nil {
			log.Debugf("Unable to close reponse body: %v", err)
		}
	}()

	if resp.StatusCode != http.StatusOK {
		if full, err := httputil.DumpResponse(resp, true); err != nil {
			log.Errorf("Could not read full response %v", err)
		} else {
			log.Errorf("Unexpected response to geo IP lookup: %v", string(full))
		}
		return "", "", fmt.Errorf("Unexpected response status %d", resp.StatusCode)
	}

	ip := resp.Header.Get("Lantern-Ip")
	country := resp.Header.Get("Lantern-Country")

	log.Debugf("Got IP and country: %v, %v", ip, country)
	return country, ip, nil
}
Example #21
0
// startupHealthcheck is used at startup to check if the server is available
// at all.
func (c *Client) startupHealthcheck(timeout time.Duration) error {
	c.mu.Lock()
	urls := c.urls
	basicAuth := c.basicAuth
	basicAuthUsername := c.basicAuthUsername
	basicAuthPassword := c.basicAuthPassword
	c.mu.Unlock()

	// If we don't get a connection after "timeout", we bail.
	start := time.Now()
	for {
		// Make a copy of the HTTP client provided via options to respect
		// settings like Basic Auth or a user-specified http.Transport.
		cl := new(http.Client)
		*cl = *c.c
		cl.Timeout = timeout

		for _, url := range urls {
			req, err := http.NewRequest("HEAD", url, nil)
			if err != nil {
				return err
			}
			if basicAuth {
				req.SetBasicAuth(basicAuthUsername, basicAuthPassword)
			}
			res, err := cl.Do(req)
			if err == nil && res != nil && res.StatusCode >= 200 && res.StatusCode < 300 {
				return nil
			}
		}
		time.Sleep(1 * time.Second)
		if time.Now().Sub(start) > timeout {
			break
		}
	}
	return ErrNoClient
}
Example #22
0
func testGates() {

	logger.Printf("Testing gates timeouts, %d gates", len(gates))
	responses := make(chan RequestWithTime, len(gates))

	for _, gate := range gates {

		client := new(http.Client)
		client.Transport = &http.Transport{Proxy: http.ProxyURL(gate.address)}
		client.Timeout = time.Duration(time.Duration(CF.GATE_TEST_TIMEOUT) * time.Second)
		gate.client = client

		go makeGetRequest(gate, TIMEOUT_TEST_URL, responses)
	}

	workingGates := []*Gate{}
	for a := 1; a <= len(gates); a++ {
		resp := <-responses
		if resp.err == nil && resp.response.StatusCode == 200 && resp.response.ContentLength >= 7 && resp.response.ContentLength <= 15 {
			contents, _ := ioutil.ReadAll(resp.response.Body)
			if contents != nil {
				//				logger.Printf("Took time - %dms, response: %s (%d)", resp.time, contents, resp.response.ContentLength)
				resp.gate.timeout = append(resp.gate.timeout, resp.time)

				resp.gate.client.Timeout = time.Duration(time.Duration(CF.GATE_REQ_TIMEOUT) * time.Second)
				workingGates = append(workingGates, resp.gate)
				fmt.Printf(".")
			}
		}
	}
	fmt.Printf("\n")
	close(responses)

	gates = workingGates

	logger.Printf("Testing gates timeouts, completed. Working gates: %d", len(gates))
}
Example #23
0
File: http.go Project: xtfly/gofd
// SendHTTPReqWithClient ...
func SendHTTPReqWithClient(client *http.Client, cfg *Config, method, addr, urlpath string, reqBody []byte) (rspBody []byte, err error) {
	schema := "http"
	if cfg.Net.TLS != nil {
		schema = "https"
	}

	if cfg.Server && !strings.Contains(addr, ":") {
		addr = fmt.Sprintf("%s:%v", addr, cfg.Net.AgentMgntPort)
	}

	url := fmt.Sprintf("%s://%s%s", schema, addr, urlpath)
	req, err := http.NewRequest(method, url, bytes.NewReader(reqBody))
	if err != nil {
		return nil, err
	}

	req.SetBasicAuth(cfg.Auth.Username, cfg.Auth.Password)
	req.Header.Set("Content-Type", "application/json")
	//log.Debugf("Sending http request %v", req)

	client.Timeout = 2 * time.Second
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode > 300 {
		return nil, fmt.Errorf("Recv http status code %v", resp.StatusCode)
	}

	if resp.ContentLength > 0 {
		rspBody, err = ioutil.ReadAll(resp.Body)
	}
	return
}
Example #24
0
// Start the Jira client goroutine
func Start(config *model.Config, wg *sync.WaitGroup,
	commandrouter *commandrouter.Router,
	outChan chan *model.ChatMessage, done chan struct{}) {

	wg.Add(1)

	httpClient := http.Client{}
	httpClient.Timeout = time.Duration(config.RequestTimeout) * time.Millisecond
	jiraClient, err := jira.NewClient(nil, config.JiraUrl)
	if err != nil {
		panic(err)
	}

	processor := &JiraCommands{
		Client: jiraClient,
	}

	incoming, err := commandrouter.AddDestination("JiraCommands", commands)
	if err != nil {
		panic(err)
	}

	go func() {
	Loop:
		for {
			select {
			case match := <-incoming:
				match.Response <- processor.Process(match.Tag, match.Message)
			case <-done:
				break Loop
			}
		}
		wg.Done()
	}()

}
Example #25
0
// RESTClientFor returns a RESTClient that satisfies the requested attributes on a client Config
// object. Note that a RESTClient may require fields that are optional when initializing a Client.
// A RESTClient created by this method is generic - it expects to operate on an API that follows
// the Kubernetes conventions, but may not be the Kubernetes API.
func RESTClientFor(config *Config) (*RESTClient, error) {
	if config.GroupVersion == nil {
		return nil, fmt.Errorf("GroupVersion is required when initializing a RESTClient")
	}
	if config.NegotiatedSerializer == nil {
		return nil, fmt.Errorf("NegotiatedSerializer is required when initializing a RESTClient")
	}
	qps := config.QPS
	if config.QPS == 0.0 {
		qps = DefaultQPS
	}
	burst := config.Burst
	if config.Burst == 0 {
		burst = DefaultBurst
	}

	baseURL, versionedAPIPath, err := defaultServerUrlFor(config)
	if err != nil {
		return nil, err
	}

	transport, err := TransportFor(config)
	if err != nil {
		return nil, err
	}

	var httpClient *http.Client
	if transport != http.DefaultTransport {
		httpClient = &http.Client{Transport: transport}
		if config.Timeout > 0 {
			httpClient.Timeout = config.Timeout
		}
	}

	return NewRESTClient(baseURL, versionedAPIPath, config.ContentConfig, qps, burst, config.RateLimiter, httpClient)
}
func (self *DefaultDownloaderMiddleware) SetClient(stop *bool, client *http.Client) {
	client.Timeout = 2 * time.Second
}
Example #27
0
func updateMirrorMappings(remoteLocation, localLocation string) bool {
	var (
		err      error
		client   http.Client
		request  *http.Request
		response *http.Response
	)
	client.Timeout = 7 * time.Second
	if request, err = http.NewRequest("GET", remoteLocation, nil); err != nil {
		log.Printf("Unable to create the request to update the mappings file. %s", err.Error())
		return false
	}
	if response, err = client.Do(request); err != nil {
		log.Printf("Unable to perform the request to update the mappings file. %s", err.Error())
		return false
	}

	var body []byte
	if body, err = ioutil.ReadAll(response.Body); err != nil {
		log.Printf("Unable to read the data to update the mappings file. %s", err.Error())
		return false
	}

	// We marshal the results into a map to make sure the results are
	// going to load correctly at startup.
	var settings map[string]*proxyConfigJSON
	if err = json.Unmarshal(body, &settings); err != nil {
		log.Printf("Unable to parse the data to update the mappings file. %s", err.Error())
		return false
	}

	tempNameBuf := make([]byte, 18)
	if _, err = rand.Read(tempNameBuf); err != nil {
		log.Printf("Unable to calculate the tempfile name to write the mappings. %s", err.Error())
		return false
	}

	tempPath := localLocation + ".tmp" + base64.URLEncoding.EncodeToString(tempNameBuf)

	var tempFile *os.File
	if tempFile, err = os.Create(tempPath); err != nil {
		log.Printf("Unable to create the tempfile to write the mappings. %s", err.Error())
		return false
	}

	jsonParser := json.NewEncoder(tempFile)
	if err = jsonParser.Encode(&settings); err != nil {
		log.Printf("Unable to write the tempfile mappings as json. %s", err.Error())
		return false
	}

	if err = tempFile.Close(); err != nil {
		log.Printf("Unable to close the tempfile for the mappings. %s", err.Error())
		return false
	}

	if err = os.Rename(tempPath, localLocation); err != nil {
		log.Printf("Unable to the mappings from tempfile to real file. %s", err.Error())
		return false
	}

	log.Printf("Updated the mirrors config.")
	return true
}
Example #28
0
func (u *UploadToLocalFile) HandFileToApp(reqTimeout time.Duration,
	respTimeout time.Duration) (ch_ret chan error) {
	u.lock.RLock()
	ch_ret = u.chHandoverWait
	u.lock.RUnlock()

	// figure out whether we have to do anything (we might have been called
	// before or we might be in a wrong state)
	u.lock_state.Lock()
	run := (u.state < StateHandingOver)
	if run {
		u.state = StateHandingOver
	}
	u.lock_state.Unlock()

	if !run {
		// if the function was called while the goroutine below is running, all
		// is fine because ch_ret will be closed when handover is done or
		// has failed
		return
	}

	go func() {
		htclient := new(http.Client)
		htclient.Timeout = reqTimeout

		// signal app backend that we are done
		v := url.Values{}
		v.Set("id", u.id)
		v.Set("filename", u.path)
		v.Set("filenameFromBrowser", u.nameFromBrowser)
		v.Set("backendSecret", u.backendSecret)
		v.Set("cancelled", "no")
		v.Set("cancelReason", "")
		u.lock.Lock()
		u.resetTimeout(u.idleTimeout)
		u.lock.Unlock()
		resp, err := htclient.PostForm(u.signalFinishURL.String(), v) // this takes time
		u.lock.Lock()
		u.resetTimeout(u.idleTimeout)
		u.lock.Unlock()

		// set error if http went through but we got a bad http status back
		if err == nil && resp.StatusCode != 200 {
			//log.Printf("Got bad http status on handover: %s", resp.Status)
			err = fmt.Errorf("Got bad http status on handover: %s", resp.Status)
		}

		// read (first 4 bytes of) response body if we can
		respBody := []byte(nil)
		if err == nil {
			if resp.ContentLength > -1 {
				respBody = make([]byte, resp.ContentLength)
				resp.Body.Read(respBody)
				resp.Body.Close()
			}
		}
		var respStr string
		if err == nil {
			respStr = string(respBody[0:4])
		}
		//log.Printf("Got response from app backend: %s", respStr)

		// response is "done"? yay, we'll be done. response is "wait"? we'll wait...
		wait := false
		if err == nil {
			if respStr == "wait" {
				wait = true
			} else if respStr == "done" {
				wait = false
			} else {
				err = errors.New("don't understand reply from app backend")
			}
		}

		// wait if we have to
		if wait {
			log.Printf("wait for app backend")
			select {
			case <-u.chHandoverDone:
				u.lock.Lock()
				u.resetTimeout(u.idleTimeout)
				u.lock.Unlock()
			case <-time.After(respTimeout):
				err = errors.New("Timed out waiting for app backend to retrieve the file")
			}
			log.Printf("wait done")
		}

		// update state
		u.lock_state.Lock()
		if err == nil {
			u.state = StateFinished
			u.lock_state.Unlock()
		} else {
			u.state = StateCancelled
			u.lock_state.Unlock()
			u.lock.RLock()
			log.Printf("upload %s handover failed: %v", u.id, err)
			u.lock.RUnlock()
			u.Cancel(false, "handover failed", 0)
		}

		// try to send error over return channel, then close it
		select {
		case ch_ret <- err:
		case <-time.After(u.idleTimeout):
		}
		close(ch_ret)
	}()
	return
}
Example #29
0
func (u *UploadToLocalFile) Cancel(tellAppBackend bool, reason string,
	reqTimeout time.Duration) error {
	u.lock.Lock()

	// set state to cancel if we can
	u.lock_state.Lock()
	alreadyCancelled := (u.state == StateCancelled)
	canCancel := (u.state < StateHandingOver)
	if canCancel {
		u.state = StateCancelled
	}
	u.lock_state.Unlock()

	// return error if we can't cancel
	if !canCancel {
		u.lock.Unlock()
		return errors.New("too late to cancel")
	}

	// close file if it is open
	if u.fd != nil {
		u.fd.Close()
		u.fd = nil
	}
	// delete file if we already have one
	if u.path != "" {
		os.Remove(u.path)
	}

	u.resetTimeout(u.idleTimeout)

	// return nil if we don't have to tell web app backend
	if alreadyCancelled || !tellAppBackend {
		u.lock.Unlock()
		return nil
	}

	// tell app backend that we have cancelled. We don't need to hold the lock
	// for this.
	backendSecret := u.backendSecret
	signalFinishURL := u.signalFinishURL
	id := u.id
	u.lock.Unlock()

	htclient := new(http.Client)
	htclient.Timeout = reqTimeout

	v := url.Values{}
	v.Set("id", id)
	v.Set("filename", "")
	v.Set("filenameFromBrowser", u.nameFromBrowser)
	v.Set("backendSecret", backendSecret)
	v.Set("cancelled", "yes")
	v.Set("cancelReason", reason)
	resp, err := htclient.PostForm(signalFinishURL.String(), v) // this takes time

	// set error if http request didn't work
	if err != nil {
		err = fmt.Errorf("http request to app backend at %s failed",
			signalFinishURL.String())
	} else {
		// set error if http went through but we got a bad http status back
		if resp.StatusCode != 200 {
			err = fmt.Errorf("Got bad http status on handover: %s", resp.Status)
		}

		// we don't care what's in the body of the response, but we read it
		// anyway so that the remote site won't suffer a broken pipe
		if resp.Body != nil {
			respBody := make([]byte, resp.ContentLength)
			resp.Body.Read(respBody)
			resp.Body.Close()
		}
	}

	u.lock.Lock()
	u.resetTimeout(u.idleTimeout)
	u.lock.Unlock()
	return err
}