func ExampleS3_HeadObject() {
svc := s3.New(nil)
params := &s3.HeadObjectInput{
Bucket: aws.String("BucketName"), // Required
Key: aws.String("ObjectKey"), // Required
IfMatch: aws.String("IfMatch"),
IfModifiedSince: aws.Time(time.Now()),
IfNoneMatch: aws.String("IfNoneMatch"),
IfUnmodifiedSince: aws.Time(time.Now()),
Range: aws.String("Range"),
RequestPayer: aws.String("RequestPayer"),
SSECustomerAlgorithm: aws.String("SSECustomerAlgorithm"),
SSECustomerKey: aws.String("SSECustomerKey"),
SSECustomerKeyMD5: aws.String("SSECustomerKeyMD5"),
VersionID: aws.String("ObjectVersionId"),
}
resp, err := svc.HeadObject(params)
if awserr := aws.Error(err); awserr != nil {
// A service error occurred.
fmt.Println("Error:", awserr.Code, awserr.Message)
} else if err != nil {
// A non-service error occurred.
panic(err)
}
// Pretty-print the response data.
fmt.Println(awsutil.StringValue(resp))
}
func validate(t *testing.T, key string, md5value string) {
svc := s3.New(nil)
resp, err := svc.GetObject(&s3.GetObjectInput{Bucket: bucketName, Key: &key})
assert.NoError(t, err)
b, _ := ioutil.ReadAll(resp.Body)
assert.Equal(t, md5value, fmt.Sprintf("%x", md5.Sum(b)))
}
func loggingSvc() (*s3.S3, *[]string, *[]interface{}) {
var m sync.Mutex
partNum := 0
names := []string{}
params := []interface{}{}
svc := s3.New(nil)
svc.Handlers.Unmarshal.Clear()
svc.Handlers.UnmarshalMeta.Clear()
svc.Handlers.UnmarshalError.Clear()
svc.Handlers.Send.Clear()
svc.Handlers.Send.PushBack(func(r *aws.Request) {
m.Lock()
defer m.Unlock()
names = append(names, r.Operation.Name)
params = append(params, r.Params)
r.HTTPResponse = &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewReader([]byte{})),
}
switch data := r.Data.(type) {
case *s3.CreateMultipartUploadOutput:
data.UploadID = aws.String("UPLOAD-ID")
case *s3.UploadPartOutput:
partNum++
data.ETag = aws.String(fmt.Sprintf("ETAG%d", partNum))
case *s3.CompleteMultipartUploadOutput:
data.Location = aws.String("https://location")
}
})
return svc, &names, ¶ms
}
// download fetches the CloudTrail logfile from S3 and parses it
func (c *config) download(m *cloudtrailNotificationRecord) (*[]cloudtrailRecord, error) {
//if len(m.S3ObjectKey) != 1 {
// return nil, fmt.Errorf("Expected one S3 key but got %d", len(m.S3ObjectKey[0]))
// }
s := s3.New(&c.awsConfig)
q := s3.GetObjectInput{
Bucket: aws.String(m.S3.Bucket.Name),
Key: aws.String(m.S3.Object.Key),
}
o, err := s.GetObject(&q)
if err != nil {
return nil, err
}
b, err := ioutil.ReadAll(o.Body)
if err != nil {
return nil, err
}
logfile := cloudtrailLog{}
if err := json.Unmarshal(b, &logfile); err != nil {
return nil, fmt.Errorf("Error unmarshaling cloutrail JSON: %s", err.Error())
}
return &logfile.Records, nil
}
func ExampleS3_DeleteObjects() {
svc := s3.New(nil)
params := &s3.DeleteObjectsInput{
Bucket: aws.String("BucketName"), // Required
Delete: &s3.Delete{ // Required
Objects: []*s3.ObjectIdentifier{ // Required
&s3.ObjectIdentifier{ // Required
Key: aws.String("ObjectKey"), // Required
VersionID: aws.String("ObjectVersionId"),
},
// More values...
},
Quiet: aws.Boolean(true),
},
MFA: aws.String("MFA"),
RequestPayer: aws.String("RequestPayer"),
}
resp, err := svc.DeleteObjects(params)
if awserr := aws.Error(err); awserr != nil {
// A service error occurred.
fmt.Println("Error:", awserr.Code, awserr.Message)
} else if err != nil {
// A non-service error occurred.
panic(err)
}
// Pretty-print the response data.
fmt.Println(awsutil.StringValue(resp))
}
func ExampleS3_CompleteMultipartUpload() {
svc := s3.New(nil)
params := &s3.CompleteMultipartUploadInput{
Bucket: aws.String("BucketName"), // Required
Key: aws.String("ObjectKey"), // Required
UploadID: aws.String("MultipartUploadId"), // Required
MultipartUpload: &s3.CompletedMultipartUpload{
Parts: []*s3.CompletedPart{
&s3.CompletedPart{ // Required
ETag: aws.String("ETag"),
PartNumber: aws.Long(1),
},
// More values...
},
},
RequestPayer: aws.String("RequestPayer"),
}
resp, err := svc.CompleteMultipartUpload(params)
if awserr := aws.Error(err); awserr != nil {
// A service error occurred.
fmt.Println("Error:", awserr.Code, awserr.Message)
} else if err != nil {
// A non-service error occurred.
panic(err)
}
// Pretty-print the response data.
fmt.Println(awsutil.StringValue(resp))
}
func (d *S3Driver) s3service() *s3.S3 {
svc := s3.New(&aws.Config{
Credentials: d.AWSCredentialsProvider,
Region: d.AWSRegion,
})
return svc
}
func ExampleS3_UploadPartCopy() {
svc := s3.New(nil)
params := &s3.UploadPartCopyInput{
Bucket: aws.String("BucketName"), // Required
CopySource: aws.String("CopySource"), // Required
Key: aws.String("ObjectKey"), // Required
PartNumber: aws.Long(1), // Required
UploadID: aws.String("MultipartUploadId"), // Required
CopySourceIfMatch: aws.String("CopySourceIfMatch"),
CopySourceIfModifiedSince: aws.Time(time.Now()),
CopySourceIfNoneMatch: aws.String("CopySourceIfNoneMatch"),
CopySourceIfUnmodifiedSince: aws.Time(time.Now()),
CopySourceRange: aws.String("CopySourceRange"),
CopySourceSSECustomerAlgorithm: aws.String("CopySourceSSECustomerAlgorithm"),
CopySourceSSECustomerKey: aws.String("CopySourceSSECustomerKey"),
CopySourceSSECustomerKeyMD5: aws.String("CopySourceSSECustomerKeyMD5"),
RequestPayer: aws.String("RequestPayer"),
SSECustomerAlgorithm: aws.String("SSECustomerAlgorithm"),
SSECustomerKey: aws.String("SSECustomerKey"),
SSECustomerKeyMD5: aws.String("SSECustomerKeyMD5"),
}
resp, err := svc.UploadPartCopy(params)
if awserr := aws.Error(err); awserr != nil {
// A service error occurred.
fmt.Println("Error:", awserr.Code, awserr.Message)
} else if err != nil {
// A non-service error occurred.
panic(err)
}
// Pretty-print the response data.
fmt.Println(awsutil.StringValue(resp))
}
func ExampleS3_CreateBucket() {
svc := s3.New(nil)
params := &s3.CreateBucketInput{
Bucket: aws.String("BucketName"), // Required
ACL: aws.String("BucketCannedACL"),
CreateBucketConfiguration: &s3.CreateBucketConfiguration{
LocationConstraint: aws.String("BucketLocationConstraint"),
},
GrantFullControl: aws.String("GrantFullControl"),
GrantRead: aws.String("GrantRead"),
GrantReadACP: aws.String("GrantReadACP"),
GrantWrite: aws.String("GrantWrite"),
GrantWriteACP: aws.String("GrantWriteACP"),
}
resp, err := svc.CreateBucket(params)
if awserr := aws.Error(err); awserr != nil {
// A service error occurred.
fmt.Println("Error:", awserr.Code, awserr.Message)
} else if err != nil {
// A non-service error occurred.
panic(err)
}
// Pretty-print the response data.
fmt.Println(awsutil.StringValue(resp))
}
func TestPresignHandler(t *testing.T) {
svc := s3.New(nil)
req, _ := svc.PutObjectRequest(&s3.PutObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
ContentDisposition: aws.String("a+b c$d"),
ACL: aws.String("public-read"),
})
req.Time = time.Unix(0, 0)
urlstr, err := req.Presign(5 * time.Minute)
assert.NoError(t, err)
expectedDate := "19700101T000000Z"
expectedHeaders := "host;x-amz-acl"
expectedSig := "7edcb4e3a1bf12f4989018d75acbe3a7f03df24bd6f3112602d59fc551f0e4e2"
expectedCred := "AKID/19700101/mock-region/s3/aws4_request"
u, _ := url.Parse(urlstr)
urlQ := u.Query()
assert.Equal(t, expectedSig, urlQ.Get("X-Amz-Signature"))
assert.Equal(t, expectedCred, urlQ.Get("X-Amz-Credential"))
assert.Equal(t, expectedHeaders, urlQ.Get("X-Amz-SignedHeaders"))
assert.Equal(t, expectedDate, urlQ.Get("X-Amz-Date"))
assert.Equal(t, "300", urlQ.Get("X-Amz-Expires"))
assert.NotContains(t, urlstr, "+") // + encoded as %20
}
func ExampleS3_PutBucketTagging() {
svc := s3.New(nil)
params := &s3.PutBucketTaggingInput{
Bucket: aws.String("BucketName"), // Required
Tagging: &s3.Tagging{ // Required
TagSet: []*s3.Tag{ // Required
&s3.Tag{ // Required
Key: aws.String("ObjectKey"), // Required
Value: aws.String("Value"), // Required
},
// More values...
},
},
ContentMD5: aws.String("ContentMD5"),
}
resp, err := svc.PutBucketTagging(params)
if awserr := aws.Error(err); awserr != nil {
// A service error occurred.
fmt.Println("Error:", awserr.Code, awserr.Message)
} else if err != nil {
// A non-service error occurred.
panic(err)
}
// Pretty-print the response data.
fmt.Println(awsutil.StringValue(resp))
}
func ExampleS3_PutBucketVersioning() {
svc := s3.New(nil)
params := &s3.PutBucketVersioningInput{
Bucket: aws.String("BucketName"), // Required
VersioningConfiguration: &s3.VersioningConfiguration{ // Required
MFADelete: aws.String("MFADelete"),
Status: aws.String("BucketVersioningStatus"),
},
ContentMD5: aws.String("ContentMD5"),
MFA: aws.String("MFA"),
}
resp, err := svc.PutBucketVersioning(params)
if awserr := aws.Error(err); awserr != nil {
// A service error occurred.
fmt.Println("Error:", awserr.Code, awserr.Message)
} else if err != nil {
// A non-service error occurred.
panic(err)
}
// Pretty-print the response data.
fmt.Println(awsutil.StringValue(resp))
}
func ExampleS3_PutBucketReplication() {
svc := s3.New(nil)
params := &s3.PutBucketReplicationInput{
Bucket: aws.String("BucketName"), // Required
ReplicationConfiguration: &s3.ReplicationConfiguration{ // Required
Role: aws.String("Role"), // Required
Rules: []*s3.ReplicationRule{ // Required
&s3.ReplicationRule{ // Required
Destination: &s3.Destination{ // Required
Bucket: aws.String("BucketName"), // Required
},
Prefix: aws.String("Prefix"), // Required
Status: aws.String("ReplicationRuleStatus"), // Required
ID: aws.String("ID"),
},
// More values...
},
},
ContentMD5: aws.String("ContentMD5"),
}
resp, err := svc.PutBucketReplication(params)
if awserr := aws.Error(err); awserr != nil {
// A service error occurred.
fmt.Println("Error:", awserr.Code, awserr.Message)
} else if err != nil {
// A non-service error occurred.
panic(err)
}
// Pretty-print the response data.
fmt.Println(awsutil.StringValue(resp))
}
// Delete stores a files with the given name at a S3 bucket.
func (s S3FileStorage) Delete(filename string) error {
cred := aws.DefaultChainCredentials
svc := s3.New(&aws.Config{Region: s.region, Credentials: cred, LogLevel: 0})
params := &s3.DeleteObjectsInput{
Bucket: aws.String(s.bucket),
Delete: &s3.Delete{
Objects: []*s3.ObjectIdentifier{
&s3.ObjectIdentifier{
Key: &filename,
},
},
Quiet: aws.Boolean(true),
},
}
_, err := svc.DeleteObjects(params)
if awserr := aws.Error(err); awserr != nil {
return err
} else if err != nil {
// A non-service error occurred.
panic(err)
}
return nil
}
func TestUploadFailCleanup(t *testing.T) {
svc := s3.New(nil)
// Break checksum on 2nd part so it fails
part := 0
svc.Handlers.Build.PushBack(func(r *aws.Request) {
if r.Operation.Name == "UploadPart" {
if part == 1 {
r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "000")
}
part++
}
})
key := "12mb-leave"
_, err := s3manager.Upload(svc, &s3manager.UploadInput{
Bucket: bucketName,
Key: &key,
Body: bytes.NewReader(integBuf12MB),
}, &s3manager.UploadOptions{
LeavePartsOnError: false,
})
assert.Error(t, err)
uploadID := ""
if merr, ok := err.(s3manager.MultiUploadFailure); ok {
uploadID = merr.UploadID()
}
assert.NotEmpty(t, uploadID)
_, err = svc.ListParts(&s3.ListPartsInput{
Bucket: bucketName, Key: &key, UploadID: &uploadID})
assert.Error(t, err)
}
func (s *S3Uploader) Upload(p *project.Project, name string, reader io.ReadSeeker, hash string) (string, string, error) {
bucketName := fmt.Sprintf("%s-%s", p.Name, someHash())
objectKey := fmt.Sprintf("%s-%s", name, hash[:12])
config := aws.DefaultConfig.Copy()
if config.Region == "" {
config.Region = "us-east-1"
}
svc := s3.New(&config)
if err := getOrCreateBucket(svc, bucketName); err != nil {
return "", "", err
}
if err := putFile(svc, bucketName, objectKey, reader); err != nil {
return "", "", err
}
req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
Bucket: &bucketName,
Key: &objectKey,
})
url, err := req.Presign(24 * 7 * time.Hour)
return objectKey, url, err
}
func ExampleS3_RestoreObject() {
svc := s3.New(nil)
params := &s3.RestoreObjectInput{
Bucket: aws.String("BucketName"), // Required
Key: aws.String("ObjectKey"), // Required
RequestPayer: aws.String("RequestPayer"),
RestoreRequest: &s3.RestoreRequest{
Days: aws.Long(1), // Required
},
VersionID: aws.String("ObjectVersionId"),
}
resp, err := svc.RestoreObject(params)
if awserr := aws.Error(err); awserr != nil {
// A service error occurred.
fmt.Println("Error:", awserr.Code, awserr.Message)
} else if err != nil {
// A non-service error occurred.
panic(err)
}
// Pretty-print the response data.
fmt.Println(awsutil.StringValue(resp))
}
func loadManager() *sneaker.Manager {
u, err := url.Parse(os.Getenv("SNEAKER_S3_PATH"))
if err != nil {
log.Fatalf("bad SNEAKER_S3_PATH: %s", err)
}
if u.Path != "" && u.Path[0] == '/' {
u.Path = u.Path[1:]
}
ctxt, err := parseContext(os.Getenv("SNEAKER_MASTER_CONTEXT"))
if err != nil {
log.Fatalf("bad SNEAKER_MASTER_CONTEXT: %s", err)
}
return &sneaker.Manager{
Objects: s3.New(nil),
Envelope: sneaker.Envelope{
KMS: kms.New(nil),
},
Bucket: u.Host,
Prefix: u.Path,
EncryptionContext: ctxt,
KeyID: os.Getenv("SNEAKER_MASTER_KEY"),
}
}
func ExampleS3_UploadPart() {
svc := s3.New(nil)
params := &s3.UploadPartInput{
Bucket: aws.String("BucketName"), // Required
Key: aws.String("ObjectKey"), // Required
PartNumber: aws.Long(1), // Required
UploadID: aws.String("MultipartUploadId"), // Required
Body: bytes.NewReader([]byte("PAYLOAD")),
ContentLength: aws.Long(1),
ContentMD5: aws.String("ContentMD5"),
RequestPayer: aws.String("RequestPayer"),
SSECustomerAlgorithm: aws.String("SSECustomerAlgorithm"),
SSECustomerKey: aws.String("SSECustomerKey"),
SSECustomerKeyMD5: aws.String("SSECustomerKeyMD5"),
}
resp, err := svc.UploadPart(params)
if awserr := aws.Error(err); awserr != nil {
// A service error occurred.
fmt.Println("Error:", awserr.Code, awserr.Message)
} else if err != nil {
// A non-service error occurred.
panic(err)
}
// Pretty-print the response data.
fmt.Println(awsutil.StringValue(resp))
}
// S3GetBucketLocation returns the AWS region of the S3 Bucket
func S3GetBucketLocation(bucket string) (string, *S3ProxyError) {
// Check if we have the bucket cached
bucketCacheItem := CacheBucketGet(bucket)
if bucketCacheItem != nil {
return bucketCacheItem.Location, nil
}
// Strange behaviour when hitting s3.amazonaws.com. Some regions work fine
// other return AuthorizationMalformedHeader. Specifying a region other than
// us-east-1 always works.
svc := s3.New(&aws.Config{Region: "eu-west-1"})
params := &s3.GetBucketLocationInput{
Bucket: aws.String(bucket),
}
resp, err := svc.GetBucketLocation(params)
if err != nil {
LogError(err)
return "", handleError(err)
}
// API returns the empty response when bucket location is US Standard
awsRegion := "us-east-1"
if resp.LocationConstraint != nil {
awsRegion = *resp.LocationConstraint
}
CacheBucketSet(bucket, awsRegion)
return awsRegion, nil
}
func ExampleS3_ListObjectVersions() {
svc := s3.New(nil)
params := &s3.ListObjectVersionsInput{
Bucket: aws.String("BucketName"), // Required
Delimiter: aws.String("Delimiter"),
EncodingType: aws.String("EncodingType"),
KeyMarker: aws.String("KeyMarker"),
MaxKeys: aws.Long(1),
Prefix: aws.String("Prefix"),
VersionIDMarker: aws.String("VersionIdMarker"),
}
resp, err := svc.ListObjectVersions(params)
if awserr := aws.Error(err); awserr != nil {
// A service error occurred.
fmt.Println("Error:", awserr.Code, awserr.Message)
} else if err != nil {
// A non-service error occurred.
panic(err)
}
// Pretty-print the response data.
fmt.Println(awsutil.StringValue(resp))
}
//NewS3Source create s3 source with access key and secret key
func NewS3Source(accessKey, secretKey string) *S3Source {
cli := s3.New(&aws.Config{
Credentials: aws.Creds(accessKey, secretKey, ""),
Region: "ap-southeast-1",
})
return &S3Source{cli}
}
func TestMD5InPutBucketPolicy(t *testing.T) {
svc := s3.New(nil)
req, _ := svc.PutBucketPolicyRequest(&s3.PutBucketPolicyInput{
Bucket: aws.String("bucketname"),
Policy: aws.String("{}"),
})
assertMD5(t, req)
}
func TestNoPopulateLocationConstraintIfClassic(t *testing.T) {
s := s3.New(&aws.Config{Region: "us-east-1"})
req, _ := s.CreateBucketRequest(&s3.CreateBucketInput{
Bucket: aws.String("bucket"),
})
err := req.Build()
assert.NoError(t, err)
assert.Equal(t, 0, len(awsutil.ValuesAtPath(req.Params, "CreateBucketConfiguration.LocationConstraint")))
}
func (s *S3Service) New() (*s3.S3, error) {
if err := os.Setenv(ENV_AWS_ACCESS_KEY, s.Keys.AccessKey); err != nil {
return nil, err
}
if err := os.Setenv(ENV_AWS_SECRET_KEY, s.Keys.SecretKey); err != nil {
return nil, err
}
return s3.New(&aws.Config{Region: s.Region}), nil
}
func TestNoPopulateLocationConstraintIfProvided(t *testing.T) {
s := s3.New(nil)
req, _ := s.CreateBucketRequest(&s3.CreateBucketInput{
Bucket: aws.String("bucket"),
CreateBucketConfiguration: &s3.CreateBucketConfiguration{},
})
err := req.Build()
assert.NoError(t, err)
assert.Equal(t, 0, len(awsutil.ValuesAtPath(req.Params, "CreateBucketConfiguration.LocationConstraint")))
}
func TestPopulateLocationConstraint(t *testing.T) {
s := s3.New(nil)
in := &s3.CreateBucketInput{
Bucket: aws.String("bucket"),
}
req, _ := s.CreateBucketRequest(in)
err := req.Build()
assert.NoError(t, err)
assert.Equal(t, "mock-region", awsutil.ValuesAtPath(req.Params, "CreateBucketConfiguration.LocationConstraint")[0])
assert.Nil(t, in.CreateBucketConfiguration) // don't modify original params
}
func TestMD5InDeleteObjects(t *testing.T) {
svc := s3.New(nil)
req, _ := svc.DeleteObjectsRequest(&s3.DeleteObjectsInput{
Bucket: aws.String("bucketname"),
Delete: &s3.Delete{
Objects: []*s3.ObjectIdentifier{
&s3.ObjectIdentifier{Key: aws.String("key")},
},
},
})
assertMD5(t, req)
}
func TestMD5InPutBucketCORS(t *testing.T) {
svc := s3.New(nil)
req, _ := svc.PutBucketCORSRequest(&s3.PutBucketCORSInput{
Bucket: aws.String("bucketname"),
CORSConfiguration: &s3.CORSConfiguration{
CORSRules: []*s3.CORSRule{
&s3.CORSRule{AllowedMethods: []*string{aws.String("GET")}},
},
},
})
assertMD5(t, req)
}
func TestMD5InPutBucketTagging(t *testing.T) {
svc := s3.New(nil)
req, _ := svc.PutBucketTaggingRequest(&s3.PutBucketTaggingInput{
Bucket: aws.String("bucketname"),
Tagging: &s3.Tagging{
TagSet: []*s3.Tag{
&s3.Tag{Key: aws.String("KEY"), Value: aws.String("VALUE")},
},
},
})
assertMD5(t, req)
}