Beispiel #1
0
func (g *S3Getter) getObject(client *s3.S3, dst, bucket, key, version string) error {
	req := &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	}
	if version != "" {
		req.VersionId = aws.String(version)
	}

	resp, err := client.GetObject(req)
	if err != nil {
		return err
	}

	// Create all the parent directories
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return err
	}

	f, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(f, resp.Body)
	return err
}
func resourceAwsS3BucketLoggingUpdate(s3conn *s3.S3, d *schema.ResourceData) error {
	logging := d.Get("logging").(*schema.Set).List()
	bucket := d.Get("bucket").(string)
	loggingStatus := &s3.BucketLoggingStatus{}

	if len(logging) > 0 {
		c := logging[0].(map[string]interface{})

		loggingEnabled := &s3.LoggingEnabled{}
		if val, ok := c["target_bucket"]; ok {
			loggingEnabled.TargetBucket = aws.String(val.(string))
		}
		if val, ok := c["target_prefix"]; ok {
			loggingEnabled.TargetPrefix = aws.String(val.(string))
		}

		loggingStatus.LoggingEnabled = loggingEnabled
	}

	i := &s3.PutBucketLoggingInput{
		Bucket:              aws.String(bucket),
		BucketLoggingStatus: loggingStatus,
	}
	log.Printf("[DEBUG] S3 put bucket logging: %#v", i)

	_, err := s3conn.PutBucketLogging(i)
	if err != nil {
		return fmt.Errorf("Error putting S3 logging: %s", err)
	}

	return nil
}
Beispiel #3
0
func (s *S3) UploadFileD(S3 *s3.S3, key string, file *os.File, contentType, perm string) (*s3.PutObjectInput, *s3.PutObjectOutput, error) {

	md5hex, _, _, err := Md5Sum(file)
	if err != nil {
		return nil, nil, err
	}
	//s.Log.Debugf("key:%s md5=%s", key, md5hex)
	req := &s3.PutObjectInput{
		ACL:    &perm,
		Body:   file,
		Bucket: &s.Bucket,
		//ContentLength: &size,
		ContentType: &contentType,
		Key:         &key,
	}
	res, err := S3.PutObject(req)
	if err != nil {
		return req, res, err
	}
	if res == nil {
		return req, res, fmt.Errorf("res is nil pointer")
	}
	if res.ETag == nil {
		return req, res, fmt.Errorf("res.ETag is nil pointer")
	}
	if len(*res.ETag) < 2 {
		return req, res, fmt.Errorf("*res.ETag is too short. It should have 2 characters or more")
	}
	etag := (*res.ETag)[1 : len(*res.ETag)-1]
	if md5hex != etag {
		return req, res, fmt.Errorf("md5 and ETag does not match. md5:%s ETag:%s", md5hex, etag)
	}
	return req, res, err
}
Beispiel #4
0
// GetS3Object returns the object output for the given object key
func GetS3Object(client *s3.S3, bucket, objKey string) (*s3.GetObjectOutput, error) {
	logger.Debug("GetS3Object: ", bucket, "-", objKey)
	params := &s3.GetObjectInput{
		Bucket: aws.String(bucket), // Required
		Key:    aws.String(objKey), // Required
		// IfMatch:                    aws.String("IfMatch"),
		// IfModifiedSince:            aws.Time(time.Now()),
		// IfNoneMatch:                aws.String("IfNoneMatch"),
		// IfUnmodifiedSince:          aws.Time(time.Now()),
		// Range:                      aws.String("Range"),
		// RequestPayer:               aws.String("RequestPayer"),
		// ResponseCacheControl:       aws.String("ResponseCacheControl"),
		// ResponseContentDisposition: aws.String("ResponseContentDisposition"),
		// ResponseContentEncoding:    aws.String("ResponseContentEncoding"),
		// ResponseContentLanguage:    aws.String("ResponseContentLanguage"),
		// ResponseContentType:        aws.String("ResponseContentType"),
		// ResponseExpires:            aws.Time(time.Now()),
		// SSECustomerAlgorithm:       aws.String("SSECustomerAlgorithm"),
		// SSECustomerKey:             aws.String("SSECustomerKey"),
		// SSECustomerKeyMD5:          aws.String("SSECustomerKeyMD5"),
		// VersionId:                  aws.String("ObjectVersionId"),
	}

	return client.GetObject(params)
}
Beispiel #5
0
func deleteObjects(bucket string, objs []Obj, wg *sync.WaitGroup, S3 *s3.S3) {
	maxLen := 1000

	for i := 0; i < len(objs); i += maxLen {
		high := i + maxLen
		if high > len(objs) {
			high = len(objs)
		}

		objects := []*s3.ObjectIdentifier{}
		for _, obj := range objs[i:high] {
			objects = append(objects, &s3.ObjectIdentifier{Key: aws.String(obj.key), VersionId: aws.String(obj.id)})
		}

		_, err := S3.DeleteObjects(&s3.DeleteObjectsInput{
			Bucket: aws.String(bucket),
			Delete: &s3.Delete{
				Objects: objects,
			},
		})
		if err != nil {
			fmt.Printf("Failed: %s\n", err)
		}

		wg.Add(-len(objects))
	}

	return
}
Beispiel #6
0
// setTags is a helper to set the tags for a resource. It expects the
// tags field to be named "tags"
func setTagsS3(conn *s3.S3, d *schema.ResourceData) error {
	if d.HasChange("tags") {
		oraw, nraw := d.GetChange("tags")
		o := oraw.(map[string]interface{})
		n := nraw.(map[string]interface{})
		create, remove := diffTagsS3(tagsFromMapS3(o), tagsFromMapS3(n))

		// Set tags
		if len(remove) > 0 {
			log.Printf("[DEBUG] Removing tags: %#v", remove)
			_, err := conn.DeleteBucketTagging(&s3.DeleteBucketTaggingInput{
				Bucket: aws.String(d.Get("bucket").(string)),
			})
			if err != nil {
				return err
			}
		}
		if len(create) > 0 {
			log.Printf("[DEBUG] Creating tags: %#v", create)
			req := &s3.PutBucketTaggingInput{
				Bucket: aws.String(d.Get("bucket").(string)),
				Tagging: &s3.Tagging{
					TagSet: create,
				},
			}

			_, err := conn.PutBucketTagging(req)
			if err != nil {
				return err
			}
		}
	}

	return nil
}
func S3ListObjectOutput(cli *s3.S3, bucket string) (listObjectOutput *s3.ListObjectsOutput, err error) {
	listObjectOutput, err = cli.ListObjects(&s3.ListObjectsInput{
		Bucket: aws.String(bucket),
	})

	return listObjectOutput, err
}
Beispiel #8
0
func pull(s3serv *s3.S3, c config, p string) {
	if list, e := s3serv.ListObjects(&s3.ListObjectsInput{Bucket: &c.Bucket}); e == nil {
		for _, l := range list.Contents {
			log.Println(*l.Key)
			if obj, e := s3serv.GetObject(&s3.GetObjectInput{Bucket: &c.Bucket, Key: l.Key}); e == nil {
				os.MkdirAll(path.Dir(p+*l.Key), 0700)
				if f, e := os.OpenFile(p+*l.Key, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666); e == nil {
					defer f.Close()
					defer obj.Body.Close()
					counter := &CountReader{input: obj.Body, co: c}
					counter.Init()
					io.Copy(f, counter)
					log.Println("written", *l.Key)
				} else {
					log.Println(e)
				}

			} else {
				log.Println(e)
			}
		}
	} else {
		log.Fatal(e)
	}

}
func listBucketObjects(svc *s3.S3, bucket string) ([]Object, []ErrObject, error) {
	listRes, err := svc.ListObjects(&s3.ListObjectsInput{
		Bucket: &bucket,
	})
	if err != nil {
		return nil, nil, err
	}

	objs := make([]Object, 0, len(listRes.Contents))
	errObjs := []ErrObject{}
	for _, listObj := range listRes.Contents {
		objData, err := svc.HeadObject(&s3.HeadObjectInput{
			Bucket: &bucket,
			Key:    listObj.Key,
		})

		if err != nil {
			errObjs = append(errObjs, ErrObject{Bucket: bucket, Key: *listObj.Key, Error: err})
			continue
		}

		obj := Object{Bucket: bucket, Key: *listObj.Key}
		if objData.ServerSideEncryption != nil {
			obj.Encrypted = true
			obj.EncryptionType = *objData.ServerSideEncryption
		}

		objs = append(objs, obj)
	}

	return objs, errObjs, nil
}
Beispiel #10
0
func recvUpload(s3serv *s3.S3, c config, files chan string) {

	for {
		path, closed := <-files

		if !closed {
			break
		}

		if f, err := os.Open(path); err == nil {
			counter := &CountReader{input: f, seeker: f, co: c}
			counter.Init()
			fmt.Print("uploading:", path)
			if _, err := s3serv.PutObject(&s3.PutObjectInput{Bucket: &c.Bucket, Key: &path, Body: counter}); err == nil {
				fmt.Println("done", counter.count)

			} else {
				fmt.Println("error uploading file:", err)
			}

		} else {
			fmt.Printf("error opening file:", err)
		}
	}
}
Beispiel #11
0
func GetRelatedProductsFromS3(svc *s3.S3, dataDir string) RelatedProducts {
	results := RelatedProducts{Relates: make(map[string]Product)}
	bucket, keypattern := parseS3Params(dataDir)
	params := &s3.ListObjectsInput{
		Bucket: aws.String("ecomm-order-items"), // Required
		//Delimiter:    aws.String("Delimiter"),
		//EncodingType: aws.String("EncodingType"),
		//Marker:       aws.String("Marker"),
		//MaxKeys:      aws.Int64(1),
		//Prefix:       aws.String("Prefix"),
	}
	resp, err := svc.ListObjects(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.

		glog.Errorf("%s, %s \n", err.(awserr.Error).Code(), err.(awserr.Error).Error())

	}

	// Pretty-print the response data.
	//fmt.Println(resp)
	for _, obj := range resp.Contents {
		glog.V(2).Infof("s3 object: %s. Keypattern:%s", *obj.Key, keypattern)
		if strings.HasPrefix(*obj.Key, keypattern[1:]+"/part-") {
			glog.V(2).Infof("populating with file %s", *obj.Key)
			populateRelatedProducts(&results, getObject(svc, bucket, *obj.Key))
		}
	}

	return results

}
func resourceAwsS3BucketVersioningUpdate(s3conn *s3.S3, d *schema.ResourceData) error {
	v := d.Get("versioning").(*schema.Set).List()
	bucket := d.Get("bucket").(string)
	vc := &s3.VersioningConfiguration{}

	if len(v) > 0 {
		c := v[0].(map[string]interface{})

		if c["enabled"].(bool) {
			vc.Status = aws.String(s3.BucketVersioningStatusEnabled)
		} else {
			vc.Status = aws.String(s3.BucketVersioningStatusSuspended)
		}
	} else {
		vc.Status = aws.String(s3.BucketVersioningStatusSuspended)
	}

	i := &s3.PutBucketVersioningInput{
		Bucket:                  aws.String(bucket),
		VersioningConfiguration: vc,
	}
	log.Printf("[DEBUG] S3 put bucket versioning: %#v", i)

	_, err := s3conn.PutBucketVersioning(i)
	if err != nil {
		return fmt.Errorf("Error putting S3 versioning: %s", err)
	}

	return nil
}
Beispiel #13
0
func head(service *s3.S3, bucketName string, key string) bool {
	_, err := service.HeadObject(&s3.HeadObjectInput{
		Bucket: stringPtr(bucketName),
		Key:    stringPtr(key),
	})
	return err == nil
}
Beispiel #14
0
func uploadFile(file string, conf config, svc *s3.S3, results chan<- result) {
	obj, err := newObjectInfo(file)
	if err != nil {
		results <- result{err: fmt.Errorf("unable to read file: %v", err)}
		return
	}

	bname := conf.Buckets[bucketName].Name
	params := &s3.PutObjectInput{
		Bucket:      &bname,
		Key:         &obj.key,
		Body:        obj.body,
		ContentType: &obj.contentType,
	}

	if !private {
		params.ACL = aws.String("public-read")
	}

	_, err = svc.PutObject(params)

	if err != nil {
		results <- result{err: err}
	} else {
		results <- result{url: buildOutputURL(obj, conf)}
	}
}
Beispiel #15
0
func ListObjects(svc *s3.S3) {

	//svc := s3.New(session.New(),&aws.Config{Region: aws.String("us-east-1")})

	params := &s3.ListObjectsInput{
		Bucket: aws.String("ecomm-order-items"), // Required
		//Delimiter:    aws.String("Delimiter"),
		//EncodingType: aws.String("EncodingType"),
		//Marker:       aws.String("Marker"),
		//MaxKeys:      aws.Int64(1),
		//Prefix:       aws.String("Prefix"),
	}
	resp, err := svc.ListObjects(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.

		fmt.Printf("%s, %s \n", err.(awserr.Error).Code(), err.(awserr.Error).Error())
		return
	}

	// Pretty-print the response data.
	//fmt.Println(resp)
	for _, obj := range resp.Contents {
		if strings.HasPrefix(*obj.Key, "recommendations/output.txt/part-") {
			fmt.Println(GetObject(svc, *obj.Key))
		}
	}
}
Beispiel #16
0
func resourceAwsS3BucketPolicyUpdate(s3conn *s3.S3, d *schema.ResourceData) error {
	bucket := d.Get("bucket").(string)
	policy := d.Get("policy").(string)

	if policy != "" {
		log.Printf("[DEBUG] S3 bucket: %s, put policy: %s", bucket, policy)

		_, err := s3conn.PutBucketPolicy(&s3.PutBucketPolicyInput{
			Bucket: aws.String(bucket),
			Policy: aws.String(policy),
		})

		if err != nil {
			return fmt.Errorf("Error putting S3 policy: %s", err)
		}
	} else {
		log.Printf("[DEBUG] S3 bucket: %s, delete policy: %s", bucket, policy)
		_, err := s3conn.DeleteBucketPolicy(&s3.DeleteBucketPolicyInput{
			Bucket: aws.String(bucket),
		})

		if err != nil {
			return fmt.Errorf("Error deleting S3 policy: %s", err)
		}
	}

	return nil
}
Beispiel #17
0
func GetObject(svc *s3.S3, key string) string {
	//svc := s3.New(session.New(),&aws.Config{Region: aws.String("us-east-1")})

	params := &s3.GetObjectInput{
		Bucket: aws.String("ecomm-order-items"), // Required
		Key:    aws.String(key),                 // Required
	}
	resp, err := svc.GetObject(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.
		glog.Fatal(err.Error())
	}

	// Pretty-print the response data.
	fmt.Print(resp)
	size := int(*resp.ContentLength)
	fmt.Println("size is ", size)
	buffer := make([]byte, size)
	defer resp.Body.Close()
	var bbuffer bytes.Buffer
	for true {

		num, rerr := resp.Body.Read(buffer)
		if num > 0 {
			//fmt.Println("times ",count)
			bbuffer.Write(buffer[:num])
			//bbuffer.WriteString(string(buffer[:num]))
		} else if rerr == io.EOF || rerr != nil {
			break
		}
	}
	return bbuffer.String()
}
Beispiel #18
0
//get object from s3
func getObject(svc *s3.S3, bucket string, key string) string {
	params := &s3.GetObjectInput{
		Bucket: aws.String(bucket), // Required
		Key:    aws.String(key),    // Required
	}
	resp, err := svc.GetObject(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.
		glog.Fatal(err.Error())
	}

	size := int(*resp.ContentLength)

	buffer := make([]byte, size)
	defer resp.Body.Close()
	var bbuffer bytes.Buffer
	for true {
		num, rerr := resp.Body.Read(buffer)
		if num > 0 {
			bbuffer.Write(buffer[:num])
		} else if rerr == io.EOF || rerr != nil {
			break
		}
	}
	return bbuffer.String()
}
func runTests(t *testing.T, svc *s3.S3, tests []s3BucketTest) {
	for _, test := range tests {
		req, _ := svc.ListObjectsRequest(&s3.ListObjectsInput{Bucket: &test.bucket})
		req.Build()
		assert.Equal(t, test.url, req.HTTPRequest.URL.String())
	}
}
func listBuckets(svc *s3.S3) ([]*Bucket, error) {
	res, err := svc.ListBuckets(&s3.ListBucketsInput{})
	if err != nil {
		return nil, err
	}

	buckets := make([]*Bucket, len(res.Buckets))
	for i, b := range res.Buckets {
		buckets[i] = &Bucket{
			Name:         *b.Name,
			CreationDate: *b.CreationDate,
		}

		locRes, err := svc.GetBucketLocation(&s3.GetBucketLocationInput{
			Bucket: b.Name,
		})
		if err != nil {
			buckets[i].Error = err
			continue
		}

		if locRes.LocationConstraint == nil {
			buckets[i].Region = "us-east-1"
		} else {
			buckets[i].Region = *locRes.LocationConstraint
		}
	}

	return buckets, nil
}
func websiteEndpoint(s3conn *s3.S3, d *schema.ResourceData) (*S3Website, error) {
	// If the bucket doesn't have a website configuration, return an empty
	// endpoint
	if _, ok := d.GetOk("website"); !ok {
		return nil, nil
	}

	bucket := d.Get("bucket").(string)

	// Lookup the region for this bucket
	location, err := s3conn.GetBucketLocation(
		&s3.GetBucketLocationInput{
			Bucket: aws.String(bucket),
		},
	)
	if err != nil {
		return nil, err
	}
	var region string
	if location.LocationConstraint != nil {
		region = *location.LocationConstraint
	}

	return WebsiteEndpoint(bucket, region), nil
}
Beispiel #22
0
func getObject(client *s3.S3, bucket string, key *string) (output *s3.GetObjectOutput, err error) {
	request := &s3.GetObjectInput{
		Bucket: &bucket,
		Key:    key,
	}

	return client.GetObject(request)
}
func resourceAwsS3BucketCorsUpdate(s3conn *s3.S3, d *schema.ResourceData) error {
	bucket := d.Get("bucket").(string)
	rawCors := d.Get("cors_rule").([]interface{})

	if len(rawCors) == 0 {
		// Delete CORS
		log.Printf("[DEBUG] S3 bucket: %s, delete CORS", bucket)
		_, err := s3conn.DeleteBucketCors(&s3.DeleteBucketCorsInput{
			Bucket: aws.String(bucket),
		})
		if err != nil {
			return fmt.Errorf("Error deleting S3 CORS: %s", err)
		}
	} else {
		// Put CORS
		rules := make([]*s3.CORSRule, 0, len(rawCors))
		for _, cors := range rawCors {
			corsMap := cors.(map[string]interface{})
			r := &s3.CORSRule{}
			for k, v := range corsMap {
				log.Printf("[DEBUG] S3 bucket: %s, put CORS: %#v, %#v", bucket, k, v)
				if k == "max_age_seconds" {
					r.MaxAgeSeconds = aws.Int64(int64(v.(int)))
				} else {
					vMap := make([]*string, len(v.([]interface{})))
					for i, vv := range v.([]interface{}) {
						str := vv.(string)
						vMap[i] = aws.String(str)
					}
					switch k {
					case "allowed_headers":
						r.AllowedHeaders = vMap
					case "allowed_methods":
						r.AllowedMethods = vMap
					case "allowed_origins":
						r.AllowedOrigins = vMap
					case "expose_headers":
						r.ExposeHeaders = vMap
					}
				}
			}
			rules = append(rules, r)
		}
		corsInput := &s3.PutBucketCorsInput{
			Bucket: aws.String(bucket),
			CORSConfiguration: &s3.CORSConfiguration{
				CORSRules: rules,
			},
		}
		log.Printf("[DEBUG] S3 bucket: %s, put CORS: %#v", bucket, corsInput)
		_, err := s3conn.PutBucketCors(corsInput)
		if err != nil {
			return fmt.Errorf("Error putting S3 CORS: %s", err)
		}
	}

	return nil
}
Beispiel #24
0
func get_object(svc *s3.S3, bucket *string, key *string) (io.ReadCloser, error) {
    params := &s3.GetObjectInput{
        Bucket: bucket,
        Key:    key,
    }
    resp, err := svc.GetObject(params)
    if err != nil { return nil, err }
    return resp.Body, nil
}
Beispiel #25
0
func s3connect(svc *s3.S3) (*s3.ListBucketsOutput, error) {
	logger.Info("Attempting to connect...")
	var params *s3.ListBucketsInput
	resp, err := svc.ListBuckets(params)
	if err != nil {
		return nil, err
	}
	return resp, nil
}
Beispiel #26
0
func makeBucket(client *s3.S3, bucket *string) error {
	cbi := &s3.CreateBucketInput{Bucket: bucket}
	_, err := client.CreateBucket(cbi)
	if err != nil {
		return err
	}

	return nil
}
Beispiel #27
0
func ObjectExists(svc *s3.S3, bucket, objName string) (bool, error) {
	_, err := svc.HeadBucket(&s3.HeadBucketInput{
		Bucket: aws.String(bucket),
	})
	if err != nil {
		return false, err
	}
	return true, nil
}
Beispiel #28
0
func deleteObject(client *s3.S3, bucket string, key *string) (err error) {
	request := &s3.DeleteObjectInput{
		Bucket: &bucket,
		Key:    key,
	}

	_, err = client.DeleteObject(request)
	return
}
Beispiel #29
0
func UploadObject(svc *s3.S3, bucketName, objKey string, reader io.Reader) error {
	params := &s3.PutObjectInput{
		Body:   aws.ReadSeekCloser(reader),
		Bucket: aws.String(bucketName),
		Key:    aws.String(objKey),
		ACL:    ACLPublicRead,
	}
	_, err := svc.PutObject(params)
	return err
}
Beispiel #30
0
func copyPart(params *s3.UploadPartCopyInput, partNumber int, client *s3.S3, notify chan<- s3.CompletedPart) {
	fmt.Println("STARTING", partNumber, *params.CopySourceRange)
	respUploadPartCopy, err1 := client.UploadPartCopy(params)
	if err1 != nil {
		fmt.Println("ERROR UploadPartCopy", err1)
		panic(err1)
	}
	fmt.Println("SUCCESSS CopyPartResult", partNumber, *respUploadPartCopy.CopyPartResult.ETag)
	notify <- s3.CompletedPart{ETag: aws.String(*respUploadPartCopy.CopyPartResult.ETag), PartNumber: aws.Long(int64(partNumber))}
}