Example #1
0
func (artifact S3Artifact) ProcessResponse(resp interface{}) (err error) {
	response := resp.(*queue.S3ArtifactResponse)
	rawContentFile := filepath.Join(taskContext.TaskDir, artifact.Base().CanonicalPath)

	// if Content-Encoding is gzip then we will need to gzip content...
	transferContentFile := rawContentFile
	if artifact.ContentEncoding == "gzip" {
		transferContentFile = gzipCompressFile(rawContentFile)
		defer os.Remove(transferContentFile)
	}

	// perform http PUT to upload to S3...
	httpClient := &http.Client{}
	httpCall := func() (*http.Response, error, error) {
		transferContent, err := os.Open(transferContentFile)
		if err != nil {
			return nil, nil, err
		}
		defer transferContent.Close()
		transferContentFileInfo, err := transferContent.Stat()
		if err != nil {
			return nil, nil, err
		}
		transferContentLength := transferContentFileInfo.Size()

		httpRequest, err := http.NewRequest("PUT", response.PutURL, transferContent)
		if err != nil {
			return nil, nil, err
		}
		httpRequest.Header.Set("Content-Type", artifact.MimeType)
		httpRequest.ContentLength = transferContentLength
		if enc := artifact.ContentEncoding; enc != "" {
			httpRequest.Header.Set("Content-Encoding", enc)
		}
		requestHeaders, dumpError := httputil.DumpRequestOut(httpRequest, false)
		if dumpError != nil {
			log.Print("Could not dump request, never mind...")
		} else {
			log.Print("Request")
			log.Print(string(requestHeaders))
		}
		putResp, err := httpClient.Do(httpRequest)
		return putResp, err, nil
	}
	putResp, putAttempts, err := httpbackoff.Retry(httpCall)
	defer putResp.Body.Close()
	log.Printf("%v put requests issued to %v", putAttempts, response.PutURL)
	respBody, dumpError := httputil.DumpResponse(putResp, true)
	if dumpError != nil {
		log.Print("Could not dump response output, never mind...")
	} else {
		log.Print("Response")
		log.Print(string(respBody))
	}
	return err
}
Example #2
0
// Request is the underlying method that makes a raw API request, without
// performing any json marshaling/unmarshaling of requests/responses. It is
// useful if you wish to handle raw payloads and/or raw http response bodies,
// rather than calling APICall which translates []byte to/from go types.
func (connectionData *ConnectionData) Request(rawPayload []byte, method, route string, query url.Values) (*CallSummary, error) {
	callSummary := new(CallSummary)
	callSummary.HTTPRequestBody = string(rawPayload)

	// function to perform http request - we call this using backoff library to
	// have exponential backoff in case of intermittent failures (e.g. network
	// blips or HTTP 5xx errors)
	httpCall := func() (*http.Response, error, error) {
		var ioReader io.Reader
		ioReader = bytes.NewReader(rawPayload)
		u, err := setURL(connectionData, route, query)
		if err != nil {
			return nil, nil, fmt.Errorf("apiCall url cannot be parsed:\n%v\n", err)
		}
		callSummary.HTTPRequest, err = http.NewRequest(method, u.String(), ioReader)
		if err != nil {
			return nil, nil, fmt.Errorf("Internal error: apiCall url cannot be parsed although thought to be valid: '%v', is the BaseURL (%v) set correctly?\n%v\n", u.String(), connectionData.BaseURL, err)
		}
		callSummary.HTTPRequest.Header.Set("Content-Type", "application/json")
		// Refresh Authorization header with each call...
		// Only authenticate if client library user wishes to.
		if connectionData.Authenticate {
			err = connectionData.Credentials.SignRequest(callSummary.HTTPRequest)
			if err != nil {
				return nil, nil, err
			}
		}
		resp, err := (&http.Client{}).Do(callSummary.HTTPRequest)
		return resp, err, nil
	}

	// Make HTTP API calls using an exponential backoff algorithm...
	var err error
	callSummary.HTTPResponse, callSummary.Attempts, err = httpbackoff.Retry(httpCall)

	// read response into memory, so that we can return the body
	if callSummary.HTTPResponse != nil {
		body, err2 := ioutil.ReadAll(callSummary.HTTPResponse.Body)
		if err2 == nil {
			callSummary.HTTPResponseBody = string(body)
		}
	}

	return callSummary, err

}
// deleteFromAzure will attempt to delete a task from the Azure queue and
// return an error in case of failure
func (q *queueService) deleteFromAzure(deleteURL string) error {
	// Messages are deleted from the Azure queue with a DELETE request to the
	// SignedDeleteURL from the Azure queue object returned from
	// queue.pollTaskURLs.

	// Also remark that the worker must delete messages if the queue.claimTask
	// operations fails with a 4xx error. A 400 hundred range error implies
	// that the task wasn't created, not scheduled or already claimed, in
	// either case the worker should delete the message as we don't want
	// another worker to receive message later.

	q.log.Info("Deleting task from Azure queue")
	httpCall := func() (*http.Response, error, error) {
		req, err := http.NewRequest("DELETE", deleteURL, nil)
		if err != nil {
			return nil, nil, err
		}
		resp, err := http.DefaultClient.Do(req)
		return resp, err, nil
	}

	_, _, err := httpbackoff.Retry(httpCall)

	// Notice, that failure to delete messages from Azure queue is serious, as
	// it wouldn't manifest itself in an immediate bug. Instead if messages
	// repeatedly fails to be deleted, it would result in a lot of unnecessary
	// calls to the queue and the Azure queue. The worker will likely continue
	// to work, as the messages eventually disappears when their deadline is
	// reached. However, the provisioner would over-provision aggressively as
	// it would be unable to tell the number of pending tasks. And the worker
	// would spend a lot of time attempting to claim faulty messages. For these
	// reasons outlined above it's strongly advised that workers logs failures
	// to delete messages from Azure queues.
	if err != nil {
		q.log.WithFields(logrus.Fields{
			"error": err,
			"url":   deleteURL,
		}).Warn("Not able to delete task from azure queue")
		return err
	}

	q.log.Info("Successfully deleted task from azure queue")
	return nil
}
// apiCall is the generic REST API calling method which performs all REST API
// calls for this library.  Each auto-generated REST API method simply is a
// wrapper around this method, calling it with specific specific arguments.
func (awsProvisioner *AwsProvisioner) apiCall(payload interface{}, method, route string, result interface{}) (interface{}, *CallSummary) {
	callSummary := new(CallSummary)
	callSummary.HttpRequestObject = payload
	var jsonPayload []byte
	jsonPayload, callSummary.Error = json.Marshal(payload)
	if callSummary.Error != nil {
		return result, callSummary
	}
	callSummary.HttpRequestBody = string(jsonPayload)

	httpClient := &http.Client{}

	// function to perform http request - we call this using backoff library to
	// have exponential backoff in case of intermittent failures (e.g. network
	// blips or HTTP 5xx errors)
	httpCall := func() (*http.Response, error, error) {
		var ioReader io.Reader = nil
		if reflect.ValueOf(payload).IsValid() && !reflect.ValueOf(payload).IsNil() {
			ioReader = bytes.NewReader(jsonPayload)
		}
		httpRequest, err := http.NewRequest(method, awsProvisioner.BaseURL+route, ioReader)
		if err != nil {
			return nil, nil, fmt.Errorf("apiCall url cannot be parsed: '%v', is your BaseURL (%v) set correctly?\n%v\n", awsProvisioner.BaseURL+route, awsProvisioner.BaseURL, err)
		}
		httpRequest.Header.Set("Content-Type", "application/json")
		callSummary.HttpRequest = httpRequest
		// Refresh Authorization header with each call...
		// Only authenticate if client library user wishes to.
		if awsProvisioner.Authenticate {
			credentials := &hawk.Credentials{
				ID:   awsProvisioner.ClientId,
				Key:  awsProvisioner.AccessToken,
				Hash: sha256.New,
			}
			reqAuth := hawk.NewRequestAuth(httpRequest, credentials, 0)
			if awsProvisioner.Certificate != "" {
				reqAuth.Ext = base64.StdEncoding.EncodeToString([]byte("{\"certificate\":" + awsProvisioner.Certificate + "}"))
			}
			httpRequest.Header.Set("Authorization", reqAuth.RequestHeader())
		}
		debug("Making http request: %v", httpRequest)
		resp, err := httpClient.Do(httpRequest)
		return resp, err, nil
	}

	// Make HTTP API calls using an exponential backoff algorithm...
	callSummary.HttpResponse, callSummary.Attempts, callSummary.Error = httpbackoff.Retry(httpCall)

	if callSummary.Error != nil {
		return result, callSummary
	}

	// now read response into memory, so that we can return the body
	var body []byte
	body, callSummary.Error = ioutil.ReadAll(callSummary.HttpResponse.Body)

	if callSummary.Error != nil {
		return result, callSummary
	}

	callSummary.HttpResponseBody = string(body)

	// if result is passed in as nil, it means the API defines no response body
	// json
	if reflect.ValueOf(result).IsValid() && !reflect.ValueOf(result).IsNil() {
		callSummary.Error = json.Unmarshal([]byte(callSummary.HttpResponseBody), &result)
		if callSummary.Error != nil {
			// technically not needed since returned outside if, but more comprehensible
			return result, callSummary
		}
	}

	// Return result and callSummary
	return result, callSummary
}
Example #5
0
func (task *TaskRun) uploadArtifact(artifact Artifact) error {
	// first check file exists!
	fileReader, err := os.Open(filepath.Join(TaskUser.HomeDir, artifact.CanonicalPath))
	if err != nil {
		return err
	}
	task.Artifacts = append(task.Artifacts, artifact)
	debug("MimeType in queue request: %v", artifact.MimeType)
	par := queue.PostArtifactRequest(json.RawMessage(`{"storageType": "s3", "expires": "` + artifact.Expires.UTC().Format("2006-01-02T15:04:05.000Z0700") + `", "contentType": "` + artifact.MimeType + `"}`))
	parsp, callSummary := Queue.CreateArtifact(
		task.TaskId,
		strconv.Itoa(int(task.RunId)),
		artifact.CanonicalPath,
		&par,
	)
	if callSummary.Error != nil {
		debug("Could not upload artifact: %v", artifact)
		debug("%v", callSummary)
		debug("%v", parsp)
		debug("Request Headers")
		callSummary.HttpRequest.Header.Write(os.Stdout)
		debug("Request Body")
		debug(callSummary.HttpRequestBody)
		debug("Response Headers")
		callSummary.HttpResponse.Header.Write(os.Stdout)
		debug("Response Body")
		debug(callSummary.HttpResponseBody)
		return callSummary.Error
	}
	debug("Response body RAW")
	debug(callSummary.HttpResponseBody)
	debug("Response body INTERPRETED")
	debug(string(*parsp))
	// unmarshal response into object
	resp := new(S3ArtifactResponse)
	err = json.Unmarshal(json.RawMessage(*parsp), resp)
	if err != nil {
		return err
	}
	httpClient := &http.Client{}
	httpCall := func() (*http.Response, error, error) {
		// instead of using fileReader, read it into memory and then use a
		// bytes.Reader since then http.NewRequest will properly set
		// Content-Length header for us, which is needed by the API we call
		requestPayload, err := ioutil.ReadAll(fileReader)
		if err != nil {
			return nil, nil, err
		}
		bytesReader := bytes.NewReader(requestPayload)
		// http.NewRequest automatically sets Content-Length correctly for bytes.Reader
		httpRequest, err := http.NewRequest("PUT", resp.PutURL, bytesReader)
		if err != nil {
			return nil, nil, err
		}
		debug("MimeType in put request: %v", artifact.MimeType)
		httpRequest.Header.Set("Content-Type", artifact.MimeType)
		// request body could be a) binary and b) massive, so don't show it...
		requestFull, dumpError := httputil.DumpRequestOut(httpRequest, false)
		if dumpError != nil {
			debug("Could not dump request, never mind...")
		} else {
			debug("Request")
			debug(string(requestFull))
		}
		putResp, err := httpClient.Do(httpRequest)
		return putResp, err, nil
	}
	putResp, putAttempts, err := httpbackoff.Retry(httpCall)
	debug("%v put requests issued to %v", putAttempts, resp.PutURL)
	respBody, dumpError := httputil.DumpResponse(putResp, true)
	if dumpError != nil {
		debug("Could not dump response output, never mind...")
	} else {
		debug("Response")
		debug(string(respBody))
	}
	return err
}