func listBucketObjects(svc *s3.S3, bucket string) ([]Object, []ErrObject, error) {
listRes, err := svc.ListObjects(&s3.ListObjectsInput{
Bucket: &bucket,
})
if err != nil {
return nil, nil, err
}
objs := make([]Object, 0, len(listRes.Contents))
errObjs := []ErrObject{}
for _, listObj := range listRes.Contents {
objData, err := svc.HeadObject(&s3.HeadObjectInput{
Bucket: &bucket,
Key: listObj.Key,
})
if err != nil {
errObjs = append(errObjs, ErrObject{Bucket: bucket, Key: *listObj.Key, Error: err})
continue
}
obj := Object{Bucket: bucket, Key: *listObj.Key}
if objData.ServerSideEncryption != nil {
obj.Encrypted = true
obj.EncryptionType = *objData.ServerSideEncryption
}
objs = append(objs, obj)
}
return objs, errObjs, nil
}
func GetRelatedProductsFromS3(svc *s3.S3, dataDir string) RelatedProducts {
results := RelatedProducts{Relates: make(map[string]Product)}
bucket, keypattern := parseS3Params(dataDir)
params := &s3.ListObjectsInput{
Bucket: aws.String("ecomm-order-items"), // Required
//Delimiter: aws.String("Delimiter"),
//EncodingType: aws.String("EncodingType"),
//Marker: aws.String("Marker"),
//MaxKeys: aws.Int64(1),
//Prefix: aws.String("Prefix"),
}
resp, err := svc.ListObjects(params)
if err != nil {
// Print the error, cast err to awserr.Error to get the Code and
// Message from an error.
glog.Errorf("%s, %s \n", err.(awserr.Error).Code(), err.(awserr.Error).Error())
}
// Pretty-print the response data.
//fmt.Println(resp)
for _, obj := range resp.Contents {
glog.V(2).Infof("s3 object: %s. Keypattern:%s", *obj.Key, keypattern)
if strings.HasPrefix(*obj.Key, keypattern[1:]+"/part-") {
glog.V(2).Infof("populating with file %s", *obj.Key)
populateRelatedProducts(&results, getObject(svc, bucket, *obj.Key))
}
}
return results
}
func ListObjects(svc *s3.S3) {
//svc := s3.New(session.New(),&aws.Config{Region: aws.String("us-east-1")})
params := &s3.ListObjectsInput{
Bucket: aws.String("ecomm-order-items"), // Required
//Delimiter: aws.String("Delimiter"),
//EncodingType: aws.String("EncodingType"),
//Marker: aws.String("Marker"),
//MaxKeys: aws.Int64(1),
//Prefix: aws.String("Prefix"),
}
resp, err := svc.ListObjects(params)
if err != nil {
// Print the error, cast err to awserr.Error to get the Code and
// Message from an error.
fmt.Printf("%s, %s \n", err.(awserr.Error).Code(), err.(awserr.Error).Error())
return
}
// Pretty-print the response data.
//fmt.Println(resp)
for _, obj := range resp.Contents {
if strings.HasPrefix(*obj.Key, "recommendations/output.txt/part-") {
fmt.Println(GetObject(svc, *obj.Key))
}
}
}
func pull(s3serv *s3.S3, c config, p string) {
if list, e := s3serv.ListObjects(&s3.ListObjectsInput{Bucket: &c.Bucket}); e == nil {
for _, l := range list.Contents {
log.Println(*l.Key)
if obj, e := s3serv.GetObject(&s3.GetObjectInput{Bucket: &c.Bucket, Key: l.Key}); e == nil {
os.MkdirAll(path.Dir(p+*l.Key), 0700)
if f, e := os.OpenFile(p+*l.Key, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666); e == nil {
defer f.Close()
defer obj.Body.Close()
counter := &CountReader{input: obj.Body, co: c}
counter.Init()
io.Copy(f, counter)
log.Println("written", *l.Key)
} else {
log.Println(e)
}
} else {
log.Println(e)
}
}
} else {
log.Fatal(e)
}
}
func S3ListObjectOutput(cli *s3.S3, bucket string) (listObjectOutput *s3.ListObjectsOutput, err error) {
listObjectOutput, err = cli.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(bucket),
})
return listObjectOutput, err
}
func emptyBucket(service *s3.S3, bucketName string) {
objs, err := service.ListObjects(&s3.ListObjectsInput{Bucket: stringPtr(bucketName)})
if err != nil {
panic(err)
}
for _, o := range objs.Contents {
_, err := service.DeleteObject(&s3.DeleteObjectInput{Bucket: stringPtr(bucketName), Key: o.Key})
if err != nil {
panic(err)
}
}
}
func listObjects(c *s3.S3, b, p string) ([]string, error) {
l := make([]string, 0)
resp, err := c.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(b),
Prefix: aws.String(p),
})
if err != nil {
return l, err
}
for _, k := range resp.Contents {
if strings.HasSuffix(*k.Key, fileSuffix) && *k.Size <= 5000 {
l = append(l, *k.Key)
}
}
return l, nil
}
func listObjects(client *s3.S3, bucket, prefix string, log *Log) (channel chan *s3.Object, err error) {
request := &s3.ListObjectsInput{
Bucket: &bucket,
Prefix: &prefix,
}
channel = make(chan *s3.Object)
if client == nil {
close(channel)
return
}
output, err := client.ListObjects(request)
if err != nil {
err = fmt.Errorf("S3 ListObjects: %s", err)
close(channel)
return
}
go func() {
defer close(channel)
for {
for i := range output.Contents {
object := output.Contents[i]
channel <- object
request.Marker = object.Key
}
if !*output.IsTruncated {
break
}
if output, err = client.ListObjects(request); err != nil {
log.Errorf("S3 ListObjects: %s", err)
break
}
}
}()
return
}
func s3MassDelete(svc *s3.S3, bucket string, key string) {
params := &s3.ListObjectsInput{
Bucket: aws.String(bucket), // Required
Prefix: aws.String(key),
}
resp, err := svc.ListObjects(params)
if err != nil {
fmt.Println(err.Error())
return
}
s3keys := []*s3.ObjectIdentifier{}
for _, m := range resp.Contents {
//log.Printf("key: %s", *m.Key)
s3keys = append(s3keys, &s3.ObjectIdentifier{Key: m.Key})
}
if len(s3keys) > 1000 {
sliced := slicer(s3keys, 5)
for _, keys := range sliced {
vals := []*s3.ObjectIdentifier{}
for _, v := range keys {
vals = append(vals, v.(*s3.ObjectIdentifier))
}
result := s3Delete(svc, bucket, vals)
if result == nil {
log.Printf("s3 result is null")
}
}
} else {
result := s3Delete(svc, bucket, s3keys)
if result == nil {
log.Printf("s3 result is null")
}
}
}