Ejemplo n.º 1
0
//CopyFileToS3 - Responsible for copying a file to S3
func (t *Task) CopyFileToS3() (err error) {
	start := time.Now()

	f := history.S3FileHistory{}
	f.Status = "error" //Changed at the end of the function if successful
	f.Bucket = rdpgs3.BucketName
	defer func() {
		f.Duration = int(time.Since(start).Seconds())
		insertErr := history.InsertS3History(f)
		if insertErr != nil {
			log.Error(fmt.Sprintf("tasks.CopyFileToS3 ! insertS3History erred : %s", err.Error()))
		}
	}()

	taskParams := []byte(t.Data)
	fm := S3FileMetadata{}
	err = json.Unmarshal(taskParams, &fm)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.CopyFileToS3() json.Unmarshal() ! %s", err))
		return err
	}

	//Log results to backups.file_history
	f.Source = fm.Location
	f.Target = fm.Location
	f.DBName = fm.DBName
	f.Node = fm.Node
	creds := credentials.NewStaticCredentials(rdpgs3.AWSAccessKey, rdpgs3.AWSSecretKey, rdpgs3.Token)

	config := &aws.Config{
		Region:           &rdpgs3.AWSRegion,
		Endpoint:         &rdpgs3.Endpoint,
		S3ForcePathStyle: &rdpgs3.S3ForcePathStyle,
		Credentials:      creds,
	}

	s3client := s3.New(config)

	file, err := os.Open(fm.Location)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.CopyFileToS3() Error attempting to open file %s ! %s", fm.Location, err))
		return err
	}

	defer file.Close()

	fileInfo, _ := file.Stat()
	f.Size = fileInfo.Size()
	f.FileName = fileInfo.Name()
	buffer := make([]byte, f.Size)
	file.Read(buffer)
	fileBytes := bytes.NewReader(buffer) // convert to io.ReadSeeker type
	fileType := http.DetectContentType(buffer)

	s3params := &s3.PutObjectInput{
		Bucket:        aws.String(rdpgs3.BucketName), // required
		Key:           aws.String(fm.Location),       // required
		ACL:           aws.String("public-read"),     //other values: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL
		Body:          fileBytes,
		ContentLength: aws.Int64(f.Size),
		ContentType:   aws.String(fileType),
		Metadata: map[string]*string{
			"Key": aws.String("MetadataValue"), //required
		},
		// see more at http://godoc.org/github.com/aws/aws-sdk-go/service/s3#S3.PutObject
	}

	result, err := s3client.PutObject(s3params)
	log.Trace(fmt.Sprintf("tasks.CopyFileToS3() Copy file to S3 result > %s ", result))

	if err != nil {
		log.Error(fmt.Sprintf("tasks.CopyFileToS3() AWS General Error ! %s", err))
		if awsErr, ok := err.(awserr.Error); ok {
			// Generic AWS Error with Code, Message, and original error (if any)
			log.Error(fmt.Sprintf("tasks.CopyFileToS3() AWS Error %s !! %s ! %s", awsErr.Code(), awsErr.Message(), awsErr.OrigErr()))
			if reqErr, ok := err.(awserr.RequestFailure); ok {
				// A service error occurred
				log.Error(fmt.Sprintf("tasks.CopyFileToS3() AWS Service Error %s !!! %s !! %s ! %s", reqErr.Code(), reqErr.Message(), reqErr.StatusCode(), reqErr.RequestID()))
			}
		} else {
			// This case should never be hit, the SDK should always return an
			// error which satisfies the awserr.Error interface.
			log.Error(fmt.Sprintf("tasks.CopyFileToS3() General AWS Error %s ! ", err.Error()))
		}
	}

	return
}
Ejemplo n.º 2
0
//Base function for getting a file from S3 and putting it somewhere on the local filesystem.
func CopyFileFromS3(source, destination, bucket string) (err error) {
	start := time.Now()
	f := history.S3FileHistory{}
	f.Status = "error" //Changed at end if successful
	spl := strings.Split(source, "/")
	basename := spl[len(spl)-1]
	//Log results to backups.file_history
	f.Source = source
	f.Target = destination
	f.Node = globals.MyIP
	f.Bucket = bucket
	f.FileName = basename
	defer func() {
		f.Duration = int(time.Since(start).Seconds())
		insertErr := history.InsertS3History(f)
		if insertErr != nil {
			log.Error(fmt.Sprintf("utils/rdpgs3.CopyFileFromS3 ! InsertS3History erred : %s", err.Error()))
		}
	}()

	if !Configured {
		errorMessage := "utils/rdpgs3.CopyFileFromS3 ! S3 Credentials not configured for this deployment!"
		log.Error(errorMessage)
		return errors.New(errorMessage)
	}

	creds := credentials.NewStaticCredentials(AWSAccessKey, AWSSecretKey, Token)

	config := &aws.Config{
		Region:           &AWSRegion,
		Endpoint:         &Endpoint,
		S3ForcePathStyle: &S3ForcePathStyle,
		Credentials:      creds,
	}

	s3client := s3.New(config)

	params := &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(source),
	}
	resp, err := s3client.GetObject(params)
	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.
		log.Error(fmt.Sprintf("tasks.StageRemoteRestore() AWS Error: ! %s", err.Error()))
		return
	}
	defer resp.Body.Close()

	pathArray := removeBlankSplit(strings.Split(destination, "/"))
	basefilename := pathArray[len(pathArray)-1]
	pathArray = pathArray[0 : len(pathArray)-1]
	if len(pathArray) == 0 {
		errorMessage := "utils/rdpgs3.CopyFileFromS3 ! Must specify destination path"
		log.Error(errorMessage)
		return errors.New(errorMessage)
	}
	folderPath := "/"
	for _, v := range pathArray {
		folderPath += (v + "/")
	}

	// make sure the folder exists before we create the file
	err = os.MkdirAll(folderPath, 0777)
	if err != nil {
		log.Error(fmt.Sprintf("utils/backup.CopyFileFromS3 ! Could not create target folder %s ! %s", folderPath, err))
		return err
	}

	downloadFile, err := os.Create(folderPath + basefilename)
	if err != nil {
		log.Error(fmt.Sprintf("utils/backup.CopyFileFromS3 ! attempting to create file error: ! %s", err))
		return err
	}

	defer downloadFile.Close()

	f.Size, err = io.Copy(downloadFile, resp.Body)

	if err != nil {
		log.Error(fmt.Sprintf("utils/backup.CopyFileFromS3 ! Failed to copy object to file ! %s", err))
		return err
	}

	f.Status = "ok"
	return err
}