func listBucketObjects(svc *s3.S3, bucket string) ([]Object, []ErrObject, error) {
listRes, err := svc.ListObjects(&s3.ListObjectsInput{
Bucket: &bucket,
})
if err != nil {
return nil, nil, err
}
objs := make([]Object, 0, len(listRes.Contents))
errObjs := []ErrObject{}
for _, listObj := range listRes.Contents {
objData, err := svc.HeadObject(&s3.HeadObjectInput{
Bucket: &bucket,
Key: listObj.Key,
})
if err != nil {
errObjs = append(errObjs, ErrObject{Bucket: bucket, Key: *listObj.Key, Error: err})
continue
}
obj := Object{Bucket: bucket, Key: *listObj.Key}
if objData.ServerSideEncryption != nil {
obj.Encrypted = true
obj.EncryptionType = *objData.ServerSideEncryption
}
objs = append(objs, obj)
}
return objs, errObjs, nil
}
func head(service *s3.S3, bucketName string, key string) bool {
_, err := service.HeadObject(&s3.HeadObjectInput{
Bucket: stringPtr(bucketName),
Key: stringPtr(key),
})
return err == nil
}
func deleteObject(client *s3.S3, bucket string, key *string) (err error) {
request := &s3.DeleteObjectInput{
Bucket: &bucket,
Key: key,
}
_, err = client.DeleteObject(request)
return
}
// GetS3Object returns the object output for the given object key
func GetS3Object(client *s3.S3, bucket, objKey string) (*s3.GetObjectOutput, error) {
logger.Debug("GetS3Object: ", bucket, "-", objKey)
params := &s3.GetObjectInput{
Bucket: aws.String(bucket), // Required
Key: aws.String(objKey), // Required
// IfMatch: aws.String("IfMatch"),
// IfModifiedSince: aws.Time(time.Now()),
// IfNoneMatch: aws.String("IfNoneMatch"),
// IfUnmodifiedSince: aws.Time(time.Now()),
// Range: aws.String("Range"),
// RequestPayer: aws.String("RequestPayer"),
// ResponseCacheControl: aws.String("ResponseCacheControl"),
// ResponseContentDisposition: aws.String("ResponseContentDisposition"),
// ResponseContentEncoding: aws.String("ResponseContentEncoding"),
// ResponseContentLanguage: aws.String("ResponseContentLanguage"),
// ResponseContentType: aws.String("ResponseContentType"),
// ResponseExpires: aws.Time(time.Now()),
// SSECustomerAlgorithm: aws.String("SSECustomerAlgorithm"),
// SSECustomerKey: aws.String("SSECustomerKey"),
// SSECustomerKeyMD5: aws.String("SSECustomerKeyMD5"),
// VersionId: aws.String("ObjectVersionId"),
}
return client.GetObject(params)
}
func (s *S3) UploadFileD(S3 *s3.S3, key string, file *os.File, contentType, perm string) (*s3.PutObjectInput, *s3.PutObjectOutput, error) {
md5hex, _, _, err := Md5Sum(file)
if err != nil {
return nil, nil, err
}
//s.Log.Debugf("key:%s md5=%s", key, md5hex)
req := &s3.PutObjectInput{
ACL: &perm,
Body: file,
Bucket: &s.Bucket,
//ContentLength: &size,
ContentType: &contentType,
Key: &key,
}
res, err := S3.PutObject(req)
if err != nil {
return req, res, err
}
if res == nil {
return req, res, fmt.Errorf("res is nil pointer")
}
if res.ETag == nil {
return req, res, fmt.Errorf("res.ETag is nil pointer")
}
if len(*res.ETag) < 2 {
return req, res, fmt.Errorf("*res.ETag is too short. It should have 2 characters or more")
}
etag := (*res.ETag)[1 : len(*res.ETag)-1]
if md5hex != etag {
return req, res, fmt.Errorf("md5 and ETag does not match. md5:%s ETag:%s", md5hex, etag)
}
return req, res, err
}
func deleteObjects(bucket string, objs []Obj, wg *sync.WaitGroup, S3 *s3.S3) {
maxLen := 1000
for i := 0; i < len(objs); i += maxLen {
high := i + maxLen
if high > len(objs) {
high = len(objs)
}
objects := []*s3.ObjectIdentifier{}
for _, obj := range objs[i:high] {
objects = append(objects, &s3.ObjectIdentifier{Key: aws.String(obj.key), VersionId: aws.String(obj.id)})
}
_, err := S3.DeleteObjects(&s3.DeleteObjectsInput{
Bucket: aws.String(bucket),
Delete: &s3.Delete{
Objects: objects,
},
})
if err != nil {
fmt.Printf("Failed: %s\n", err)
}
wg.Add(-len(objects))
}
return
}
func (g *S3Getter) getObject(client *s3.S3, dst, bucket, key, version string) error {
req := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}
if version != "" {
req.VersionId = aws.String(version)
}
resp, err := client.GetObject(req)
if err != nil {
return err
}
// Create all the parent directories
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return err
}
f, err := os.Create(dst)
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(f, resp.Body)
return err
}
func resourceAwsS3BucketLoggingUpdate(s3conn *s3.S3, d *schema.ResourceData) error {
logging := d.Get("logging").(*schema.Set).List()
bucket := d.Get("bucket").(string)
loggingStatus := &s3.BucketLoggingStatus{}
if len(logging) > 0 {
c := logging[0].(map[string]interface{})
loggingEnabled := &s3.LoggingEnabled{}
if val, ok := c["target_bucket"]; ok {
loggingEnabled.TargetBucket = aws.String(val.(string))
}
if val, ok := c["target_prefix"]; ok {
loggingEnabled.TargetPrefix = aws.String(val.(string))
}
loggingStatus.LoggingEnabled = loggingEnabled
}
i := &s3.PutBucketLoggingInput{
Bucket: aws.String(bucket),
BucketLoggingStatus: loggingStatus,
}
log.Printf("[DEBUG] S3 put bucket logging: %#v", i)
_, err := s3conn.PutBucketLogging(i)
if err != nil {
return fmt.Errorf("Error putting S3 logging: %s", err)
}
return nil
}
func recvUpload(s3serv *s3.S3, c config, files chan string) {
for {
path, closed := <-files
if !closed {
break
}
if f, err := os.Open(path); err == nil {
counter := &CountReader{input: f, seeker: f, co: c}
counter.Init()
fmt.Print("uploading:", path)
if _, err := s3serv.PutObject(&s3.PutObjectInput{Bucket: &c.Bucket, Key: &path, Body: counter}); err == nil {
fmt.Println("done", counter.count)
} else {
fmt.Println("error uploading file:", err)
}
} else {
fmt.Printf("error opening file:", err)
}
}
}
func S3ListObjectOutput(cli *s3.S3, bucket string) (listObjectOutput *s3.ListObjectsOutput, err error) {
listObjectOutput, err = cli.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(bucket),
})
return listObjectOutput, err
}
func listBuckets(svc *s3.S3) ([]*Bucket, error) {
res, err := svc.ListBuckets(&s3.ListBucketsInput{})
if err != nil {
return nil, err
}
buckets := make([]*Bucket, len(res.Buckets))
for i, b := range res.Buckets {
buckets[i] = &Bucket{
Name: *b.Name,
CreationDate: *b.CreationDate,
}
locRes, err := svc.GetBucketLocation(&s3.GetBucketLocationInput{
Bucket: b.Name,
})
if err != nil {
buckets[i].Error = err
continue
}
if locRes.LocationConstraint == nil {
buckets[i].Region = "us-east-1"
} else {
buckets[i].Region = *locRes.LocationConstraint
}
}
return buckets, nil
}
//get object from s3
func getObject(svc *s3.S3, bucket string, key string) string {
params := &s3.GetObjectInput{
Bucket: aws.String(bucket), // Required
Key: aws.String(key), // Required
}
resp, err := svc.GetObject(params)
if err != nil {
// Print the error, cast err to awserr.Error to get the Code and
// Message from an error.
glog.Fatal(err.Error())
}
size := int(*resp.ContentLength)
buffer := make([]byte, size)
defer resp.Body.Close()
var bbuffer bytes.Buffer
for true {
num, rerr := resp.Body.Read(buffer)
if num > 0 {
bbuffer.Write(buffer[:num])
} else if rerr == io.EOF || rerr != nil {
break
}
}
return bbuffer.String()
}
func GetObject(svc *s3.S3, key string) string {
//svc := s3.New(session.New(),&aws.Config{Region: aws.String("us-east-1")})
params := &s3.GetObjectInput{
Bucket: aws.String("ecomm-order-items"), // Required
Key: aws.String(key), // Required
}
resp, err := svc.GetObject(params)
if err != nil {
// Print the error, cast err to awserr.Error to get the Code and
// Message from an error.
glog.Fatal(err.Error())
}
// Pretty-print the response data.
fmt.Print(resp)
size := int(*resp.ContentLength)
fmt.Println("size is ", size)
buffer := make([]byte, size)
defer resp.Body.Close()
var bbuffer bytes.Buffer
for true {
num, rerr := resp.Body.Read(buffer)
if num > 0 {
//fmt.Println("times ",count)
bbuffer.Write(buffer[:num])
//bbuffer.WriteString(string(buffer[:num]))
} else if rerr == io.EOF || rerr != nil {
break
}
}
return bbuffer.String()
}
func websiteEndpoint(s3conn *s3.S3, d *schema.ResourceData) (*S3Website, error) {
// If the bucket doesn't have a website configuration, return an empty
// endpoint
if _, ok := d.GetOk("website"); !ok {
return nil, nil
}
bucket := d.Get("bucket").(string)
// Lookup the region for this bucket
location, err := s3conn.GetBucketLocation(
&s3.GetBucketLocationInput{
Bucket: aws.String(bucket),
},
)
if err != nil {
return nil, err
}
var region string
if location.LocationConstraint != nil {
region = *location.LocationConstraint
}
return WebsiteEndpoint(bucket, region), nil
}
func runTests(t *testing.T, svc *s3.S3, tests []s3BucketTest) {
for _, test := range tests {
req, _ := svc.ListObjectsRequest(&s3.ListObjectsInput{Bucket: &test.bucket})
req.Build()
assert.Equal(t, test.url, req.HTTPRequest.URL.String())
}
}
func uploadFile(file string, conf config, svc *s3.S3, results chan<- result) {
obj, err := newObjectInfo(file)
if err != nil {
results <- result{err: fmt.Errorf("unable to read file: %v", err)}
return
}
bname := conf.Buckets[bucketName].Name
params := &s3.PutObjectInput{
Bucket: &bname,
Key: &obj.key,
Body: obj.body,
ContentType: &obj.contentType,
}
if !private {
params.ACL = aws.String("public-read")
}
_, err = svc.PutObject(params)
if err != nil {
results <- result{err: err}
} else {
results <- result{url: buildOutputURL(obj, conf)}
}
}
// setTags is a helper to set the tags for a resource. It expects the
// tags field to be named "tags"
func setTagsS3(conn *s3.S3, d *schema.ResourceData) error {
if d.HasChange("tags") {
oraw, nraw := d.GetChange("tags")
o := oraw.(map[string]interface{})
n := nraw.(map[string]interface{})
create, remove := diffTagsS3(tagsFromMapS3(o), tagsFromMapS3(n))
// Set tags
if len(remove) > 0 {
log.Printf("[DEBUG] Removing tags: %#v", remove)
_, err := conn.DeleteBucketTagging(&s3.DeleteBucketTaggingInput{
Bucket: aws.String(d.Get("bucket").(string)),
})
if err != nil {
return err
}
}
if len(create) > 0 {
log.Printf("[DEBUG] Creating tags: %#v", create)
req := &s3.PutBucketTaggingInput{
Bucket: aws.String(d.Get("bucket").(string)),
Tagging: &s3.Tagging{
TagSet: create,
},
}
_, err := conn.PutBucketTagging(req)
if err != nil {
return err
}
}
}
return nil
}
func resourceAwsS3BucketVersioningUpdate(s3conn *s3.S3, d *schema.ResourceData) error {
v := d.Get("versioning").(*schema.Set).List()
bucket := d.Get("bucket").(string)
vc := &s3.VersioningConfiguration{}
if len(v) > 0 {
c := v[0].(map[string]interface{})
if c["enabled"].(bool) {
vc.Status = aws.String(s3.BucketVersioningStatusEnabled)
} else {
vc.Status = aws.String(s3.BucketVersioningStatusSuspended)
}
} else {
vc.Status = aws.String(s3.BucketVersioningStatusSuspended)
}
i := &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket),
VersioningConfiguration: vc,
}
log.Printf("[DEBUG] S3 put bucket versioning: %#v", i)
_, err := s3conn.PutBucketVersioning(i)
if err != nil {
return fmt.Errorf("Error putting S3 versioning: %s", err)
}
return nil
}
func resourceAwsS3BucketPolicyUpdate(s3conn *s3.S3, d *schema.ResourceData) error {
bucket := d.Get("bucket").(string)
policy := d.Get("policy").(string)
if policy != "" {
log.Printf("[DEBUG] S3 bucket: %s, put policy: %s", bucket, policy)
_, err := s3conn.PutBucketPolicy(&s3.PutBucketPolicyInput{
Bucket: aws.String(bucket),
Policy: aws.String(policy),
})
if err != nil {
return fmt.Errorf("Error putting S3 policy: %s", err)
}
} else {
log.Printf("[DEBUG] S3 bucket: %s, delete policy: %s", bucket, policy)
_, err := s3conn.DeleteBucketPolicy(&s3.DeleteBucketPolicyInput{
Bucket: aws.String(bucket),
})
if err != nil {
return fmt.Errorf("Error deleting S3 policy: %s", err)
}
}
return nil
}
func ListObjects(svc *s3.S3) {
//svc := s3.New(session.New(),&aws.Config{Region: aws.String("us-east-1")})
params := &s3.ListObjectsInput{
Bucket: aws.String("ecomm-order-items"), // Required
//Delimiter: aws.String("Delimiter"),
//EncodingType: aws.String("EncodingType"),
//Marker: aws.String("Marker"),
//MaxKeys: aws.Int64(1),
//Prefix: aws.String("Prefix"),
}
resp, err := svc.ListObjects(params)
if err != nil {
// Print the error, cast err to awserr.Error to get the Code and
// Message from an error.
fmt.Printf("%s, %s \n", err.(awserr.Error).Code(), err.(awserr.Error).Error())
return
}
// Pretty-print the response data.
//fmt.Println(resp)
for _, obj := range resp.Contents {
if strings.HasPrefix(*obj.Key, "recommendations/output.txt/part-") {
fmt.Println(GetObject(svc, *obj.Key))
}
}
}
func GetRelatedProductsFromS3(svc *s3.S3, dataDir string) RelatedProducts {
results := RelatedProducts{Relates: make(map[string]Product)}
bucket, keypattern := parseS3Params(dataDir)
params := &s3.ListObjectsInput{
Bucket: aws.String("ecomm-order-items"), // Required
//Delimiter: aws.String("Delimiter"),
//EncodingType: aws.String("EncodingType"),
//Marker: aws.String("Marker"),
//MaxKeys: aws.Int64(1),
//Prefix: aws.String("Prefix"),
}
resp, err := svc.ListObjects(params)
if err != nil {
// Print the error, cast err to awserr.Error to get the Code and
// Message from an error.
glog.Errorf("%s, %s \n", err.(awserr.Error).Code(), err.(awserr.Error).Error())
}
// Pretty-print the response data.
//fmt.Println(resp)
for _, obj := range resp.Contents {
glog.V(2).Infof("s3 object: %s. Keypattern:%s", *obj.Key, keypattern)
if strings.HasPrefix(*obj.Key, keypattern[1:]+"/part-") {
glog.V(2).Infof("populating with file %s", *obj.Key)
populateRelatedProducts(&results, getObject(svc, bucket, *obj.Key))
}
}
return results
}
func pull(s3serv *s3.S3, c config, p string) {
if list, e := s3serv.ListObjects(&s3.ListObjectsInput{Bucket: &c.Bucket}); e == nil {
for _, l := range list.Contents {
log.Println(*l.Key)
if obj, e := s3serv.GetObject(&s3.GetObjectInput{Bucket: &c.Bucket, Key: l.Key}); e == nil {
os.MkdirAll(path.Dir(p+*l.Key), 0700)
if f, e := os.OpenFile(p+*l.Key, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666); e == nil {
defer f.Close()
defer obj.Body.Close()
counter := &CountReader{input: obj.Body, co: c}
counter.Init()
io.Copy(f, counter)
log.Println("written", *l.Key)
} else {
log.Println(e)
}
} else {
log.Println(e)
}
}
} else {
log.Fatal(e)
}
}
func getObject(client *s3.S3, bucket string, key *string) (output *s3.GetObjectOutput, err error) {
request := &s3.GetObjectInput{
Bucket: &bucket,
Key: key,
}
return client.GetObject(request)
}
func resourceAwsS3BucketCorsUpdate(s3conn *s3.S3, d *schema.ResourceData) error {
bucket := d.Get("bucket").(string)
rawCors := d.Get("cors_rule").([]interface{})
if len(rawCors) == 0 {
// Delete CORS
log.Printf("[DEBUG] S3 bucket: %s, delete CORS", bucket)
_, err := s3conn.DeleteBucketCors(&s3.DeleteBucketCorsInput{
Bucket: aws.String(bucket),
})
if err != nil {
return fmt.Errorf("Error deleting S3 CORS: %s", err)
}
} else {
// Put CORS
rules := make([]*s3.CORSRule, 0, len(rawCors))
for _, cors := range rawCors {
corsMap := cors.(map[string]interface{})
r := &s3.CORSRule{}
for k, v := range corsMap {
log.Printf("[DEBUG] S3 bucket: %s, put CORS: %#v, %#v", bucket, k, v)
if k == "max_age_seconds" {
r.MaxAgeSeconds = aws.Int64(int64(v.(int)))
} else {
vMap := make([]*string, len(v.([]interface{})))
for i, vv := range v.([]interface{}) {
str := vv.(string)
vMap[i] = aws.String(str)
}
switch k {
case "allowed_headers":
r.AllowedHeaders = vMap
case "allowed_methods":
r.AllowedMethods = vMap
case "allowed_origins":
r.AllowedOrigins = vMap
case "expose_headers":
r.ExposeHeaders = vMap
}
}
}
rules = append(rules, r)
}
corsInput := &s3.PutBucketCorsInput{
Bucket: aws.String(bucket),
CORSConfiguration: &s3.CORSConfiguration{
CORSRules: rules,
},
}
log.Printf("[DEBUG] S3 bucket: %s, put CORS: %#v", bucket, corsInput)
_, err := s3conn.PutBucketCors(corsInput)
if err != nil {
return fmt.Errorf("Error putting S3 CORS: %s", err)
}
}
return nil
}
func makeBucket(client *s3.S3, bucket *string) error {
cbi := &s3.CreateBucketInput{Bucket: bucket}
_, err := client.CreateBucket(cbi)
if err != nil {
return err
}
return nil
}
func ObjectExists(svc *s3.S3, bucket, objName string) (bool, error) {
_, err := svc.HeadBucket(&s3.HeadBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
return false, err
}
return true, nil
}
func get_object(svc *s3.S3, bucket *string, key *string) (io.ReadCloser, error) {
params := &s3.GetObjectInput{
Bucket: bucket,
Key: key,
}
resp, err := svc.GetObject(params)
if err != nil { return nil, err }
return resp.Body, nil
}
func s3connect(svc *s3.S3) (*s3.ListBucketsOutput, error) {
logger.Info("Attempting to connect...")
var params *s3.ListBucketsInput
resp, err := svc.ListBuckets(params)
if err != nil {
return nil, err
}
return resp, nil
}
func copyPart(params *s3.UploadPartCopyInput, partNumber int, client *s3.S3, notify chan<- s3.CompletedPart) {
fmt.Println("STARTING", partNumber, *params.CopySourceRange)
respUploadPartCopy, err1 := client.UploadPartCopy(params)
if err1 != nil {
fmt.Println("ERROR UploadPartCopy", err1)
panic(err1)
}
fmt.Println("SUCCESSS CopyPartResult", partNumber, *respUploadPartCopy.CopyPartResult.ETag)
notify <- s3.CompletedPart{ETag: aws.String(*respUploadPartCopy.CopyPartResult.ETag), PartNumber: aws.Long(int64(partNumber))}
}
func UploadObject(svc *s3.S3, bucketName, objKey string, reader io.Reader) error {
params := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(reader),
Bucket: aws.String(bucketName),
Key: aws.String(objKey),
ACL: ACLPublicRead,
}
_, err := svc.PutObject(params)
return err
}