//CopyFileToS3 - Responsible for copying a file to S3
func (t *Task) CopyFileToS3() (err error) {
start := time.Now()
f := history.S3FileHistory{}
f.Status = "error" //Changed at the end of the function if successful
f.Bucket = rdpgs3.BucketName
defer func() {
f.Duration = int(time.Since(start).Seconds())
insertErr := history.InsertS3History(f)
if insertErr != nil {
log.Error(fmt.Sprintf("tasks.CopyFileToS3 ! insertS3History erred : %s", err.Error()))
}
}()
taskParams := []byte(t.Data)
fm := S3FileMetadata{}
err = json.Unmarshal(taskParams, &fm)
if err != nil {
log.Error(fmt.Sprintf("tasks.CopyFileToS3() json.Unmarshal() ! %s", err))
return err
}
//Log results to backups.file_history
f.Source = fm.Location
f.Target = fm.Location
f.DBName = fm.DBName
f.Node = fm.Node
creds := credentials.NewStaticCredentials(rdpgs3.AWSAccessKey, rdpgs3.AWSSecretKey, rdpgs3.Token)
config := &aws.Config{
Region: &rdpgs3.AWSRegion,
Endpoint: &rdpgs3.Endpoint,
S3ForcePathStyle: &rdpgs3.S3ForcePathStyle,
Credentials: creds,
}
s3client := s3.New(config)
file, err := os.Open(fm.Location)
if err != nil {
log.Error(fmt.Sprintf("tasks.CopyFileToS3() Error attempting to open file %s ! %s", fm.Location, err))
return err
}
defer file.Close()
fileInfo, _ := file.Stat()
f.Size = fileInfo.Size()
f.FileName = fileInfo.Name()
buffer := make([]byte, f.Size)
file.Read(buffer)
fileBytes := bytes.NewReader(buffer) // convert to io.ReadSeeker type
fileType := http.DetectContentType(buffer)
s3params := &s3.PutObjectInput{
Bucket: aws.String(rdpgs3.BucketName), // required
Key: aws.String(fm.Location), // required
ACL: aws.String("public-read"), //other values: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL
Body: fileBytes,
ContentLength: aws.Int64(f.Size),
ContentType: aws.String(fileType),
Metadata: map[string]*string{
"Key": aws.String("MetadataValue"), //required
},
// see more at http://godoc.org/github.com/aws/aws-sdk-go/service/s3#S3.PutObject
}
result, err := s3client.PutObject(s3params)
log.Trace(fmt.Sprintf("tasks.CopyFileToS3() Copy file to S3 result > %s ", result))
if err != nil {
log.Error(fmt.Sprintf("tasks.CopyFileToS3() AWS General Error ! %s", err))
if awsErr, ok := err.(awserr.Error); ok {
// Generic AWS Error with Code, Message, and original error (if any)
log.Error(fmt.Sprintf("tasks.CopyFileToS3() AWS Error %s !! %s ! %s", awsErr.Code(), awsErr.Message(), awsErr.OrigErr()))
if reqErr, ok := err.(awserr.RequestFailure); ok {
// A service error occurred
log.Error(fmt.Sprintf("tasks.CopyFileToS3() AWS Service Error %s !!! %s !! %s ! %s", reqErr.Code(), reqErr.Message(), reqErr.StatusCode(), reqErr.RequestID()))
}
} else {
// This case should never be hit, the SDK should always return an
// error which satisfies the awserr.Error interface.
log.Error(fmt.Sprintf("tasks.CopyFileToS3() General AWS Error %s ! ", err.Error()))
}
}
return
}
//Base function for getting a file from S3 and putting it somewhere on the local filesystem.
func CopyFileFromS3(source, destination, bucket string) (err error) {
start := time.Now()
f := history.S3FileHistory{}
f.Status = "error" //Changed at end if successful
spl := strings.Split(source, "/")
basename := spl[len(spl)-1]
//Log results to backups.file_history
f.Source = source
f.Target = destination
f.Node = globals.MyIP
f.Bucket = bucket
f.FileName = basename
defer func() {
f.Duration = int(time.Since(start).Seconds())
insertErr := history.InsertS3History(f)
if insertErr != nil {
log.Error(fmt.Sprintf("utils/rdpgs3.CopyFileFromS3 ! InsertS3History erred : %s", err.Error()))
}
}()
if !Configured {
errorMessage := "utils/rdpgs3.CopyFileFromS3 ! S3 Credentials not configured for this deployment!"
log.Error(errorMessage)
return errors.New(errorMessage)
}
creds := credentials.NewStaticCredentials(AWSAccessKey, AWSSecretKey, Token)
config := &aws.Config{
Region: &AWSRegion,
Endpoint: &Endpoint,
S3ForcePathStyle: &S3ForcePathStyle,
Credentials: creds,
}
s3client := s3.New(config)
params := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(source),
}
resp, err := s3client.GetObject(params)
if err != nil {
// Print the error, cast err to awserr.Error to get the Code and
// Message from an error.
log.Error(fmt.Sprintf("tasks.StageRemoteRestore() AWS Error: ! %s", err.Error()))
return
}
defer resp.Body.Close()
pathArray := removeBlankSplit(strings.Split(destination, "/"))
basefilename := pathArray[len(pathArray)-1]
pathArray = pathArray[0 : len(pathArray)-1]
if len(pathArray) == 0 {
errorMessage := "utils/rdpgs3.CopyFileFromS3 ! Must specify destination path"
log.Error(errorMessage)
return errors.New(errorMessage)
}
folderPath := "/"
for _, v := range pathArray {
folderPath += (v + "/")
}
// make sure the folder exists before we create the file
err = os.MkdirAll(folderPath, 0777)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.CopyFileFromS3 ! Could not create target folder %s ! %s", folderPath, err))
return err
}
downloadFile, err := os.Create(folderPath + basefilename)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.CopyFileFromS3 ! attempting to create file error: ! %s", err))
return err
}
defer downloadFile.Close()
f.Size, err = io.Copy(downloadFile, resp.Body)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.CopyFileFromS3 ! Failed to copy object to file ! %s", err))
return err
}
f.Status = "ok"
return err
}