func createKeys(t *testing.T, bucket *Bucket, keys ...keyMeta) []keyMeta {
for _, key := range keys {
req := s3.PutObjectInput{
Body: bytes.NewReader([]byte("Data")),
Bucket: aws.String(bucket.Name),
Key: aws.String(key.key),
}
if key.meta != nil {
req.Metadata = map[string]*string{}
for k, v := range key.meta {
req.Metadata[k] = &v
}
}
if _, err := bucket.Client.PutObject(&req); err != nil {
t.Fatalf("Error while creating key %s in %s: %s", key, bucket.Name, err)
}
}
return keys
}
// writeToS3 uploads the given file to S3
func writeToS3(s3Conn s3Connection, input io.ReadSeeker, disableEncryption bool) error {
params := s3.PutObjectInput{
Bucket: aws.String(s3Conn.bucket),
Key: aws.String(s3Conn.key),
Body: input,
}
if !disableEncryption {
algo := aesAlgo
params.ServerSideEncryption = &algo
}
_, err := s3Conn.handler.PutObject(¶ms)
return err
}
// PutObjectRequest creates a temp file to encrypt the contents into. It then streams
// that data to S3.
//
// Example:
// svc := s3crypto.New(session.New(), s3crypto.AESGCMContentCipherBuilder(handler))
// req, out := svc.PutObjectRequest(&s3.PutObjectInput {
// Key: aws.String("testKey"),
// Bucket: aws.String("testBucket"),
// Body: bytes.NewBuffer("test data"),
// })
// err := req.Send()
func (c *EncryptionClient) PutObjectRequest(input *s3.PutObjectInput) (*request.Request, *s3.PutObjectOutput) {
req, out := c.S3Client.PutObjectRequest(input)
// Get Size of file
n, err := input.Body.Seek(0, 2)
if err != nil {
req.Error = err
return req, out
}
input.Body.Seek(0, 0)
dst, err := getWriterStore(req, c.TempFolderPath, n >= c.MinFileSize)
if err != nil {
req.Error = err
return req, out
}
encryptor, err := c.ContentCipherBuilder.ContentCipher()
req.Handlers.Build.PushFront(func(r *request.Request) {
if err != nil {
r.Error = err
return
}
md5 := newMD5Reader(input.Body)
sha := newSHA256Writer(dst)
reader, err := encryptor.EncryptContents(md5)
if err != nil {
r.Error = err
return
}
_, err = io.Copy(sha, reader)
if err != nil {
r.Error = err
return
}
data := encryptor.GetCipherData()
env, err := encodeMeta(md5, data)
if err != nil {
r.Error = err
return
}
shaHex := hex.EncodeToString(sha.GetValue())
req.HTTPRequest.Header.Set("X-Amz-Content-Sha256", shaHex)
dst.Seek(0, 0)
input.Body = dst
err = c.SaveStrategy.Save(env, r)
r.Error = err
})
return req, out
}
// Save will save the envelope contents to s3.
func (strat S3SaveStrategy) Save(env Envelope, req *request.Request) error {
input := req.Params.(*s3.PutObjectInput)
b, err := json.Marshal(env)
if err != nil {
return err
}
instInput := s3.PutObjectInput{
Bucket: input.Bucket,
Body: bytes.NewReader(b),
}
if strat.InstructionFileSuffix == "" {
instInput.Key = aws.String(*input.Key + DefaultInstructionKeySuffix)
} else {
instInput.Key = aws.String(*input.Key + strat.InstructionFileSuffix)
}
_, err = strat.Client.PutObject(&instInput)
return err
}
func (s *S3Suite) TestS3ModifyDuringRead(c *C) {
createBucket := s3.CreateBucketInput{Bucket: aws.String("modified")}
_, err := s.svc.CreateBucket(&createBucket)
c.Assert(err, IsNil)
// upload a file
buffer := bytes.NewReader(make([]byte, 100))
putObject := s3.PutObjectInput{}
putObject.Bucket = aws.String("modified")
putObject.Key = aws.String("prefix/banana")
putObject.Body = buffer
s.svc.PutObject(&putObject)
// make sure ListDir sees it
conn := NewS3Connection(credentials.AnonymousCredentials, "modified", "prefix", s.region, s.endpoint)
files, err := conn.ListDir("", nil)
c.Assert(err, IsNil)
c.Assert(len(files.Files), Equals, 1)
f := files.Files[0]
c.Assert(f.Name, Equals, "banana")
localFile, _ := ioutil.TempFile(c.MkDir(), "local")
localPath := localFile.Name()
localFile.Close()
status := &NullStatusCallback{}
// Perform a read
region, err := conn.PrepareForRead("banana", f.Etag, localPath, 0, 10, status)
c.Assert(err, IsNil)
c.Assert(region.Offset, Equals, uint64(0))
c.Assert(region.Length, Equals, uint64(10))
// upload new version of object
buffer = bytes.NewReader(make([]byte, 101))
putObject = s3.PutObjectInput{}
putObject.Bucket = aws.String("modified")
putObject.Key = aws.String("prefix/banana")
putObject.Body = buffer
s.svc.PutObject(&putObject)
// try a read, and we should get a failure because data changed, and hence Etag
_, err = conn.PrepareForRead("banana", f.Etag, localPath, 10, 20, status)
c.Assert(err, Equals, UpdateDetected)
}
func (s *S3Suite) TestS3Connection(c *C) {
createBucket := s3.CreateBucketInput{Bucket: aws.String("bucket")}
_, err := s.svc.CreateBucket(&createBucket)
c.Assert(err, IsNil)
conn := NewS3Connection(credentials.AnonymousCredentials, "bucket", "prefix", s.region, s.endpoint)
files, err := conn.ListDir("path", nil)
c.Assert(err, IsNil)
c.Assert(len(files.Files), Equals, 0)
buffer := bytes.NewReader(make([]byte, 100))
putObject := s3.PutObjectInput{}
putObject.Bucket = aws.String("bucket")
putObject.Key = aws.String("prefix/banana")
putObject.Body = buffer
s.svc.PutObject(&putObject)
putObject = s3.PutObjectInput{}
putObject.Bucket = aws.String("bucket")
putObject.Key = aws.String("prefix/sampledir/")
putObject.Body = bytes.NewReader(make([]byte, 0))
s.svc.PutObject(&putObject)
putObject = s3.PutObjectInput{}
putObject.Bucket = aws.String("bucket")
putObject.Key = aws.String("prefix/sampledir/a")
putObject.Body = bytes.NewReader(make([]byte, 0))
s.svc.PutObject(&putObject)
putObject = s3.PutObjectInput{}
putObject.Bucket = aws.String("bucket")
putObject.Key = aws.String("prefix/sampledir/b")
putObject.Body = bytes.NewReader(make([]byte, 0))
s.svc.PutObject(&putObject)
files, err = conn.ListDir("", nil)
fmt.Printf("files=%s\n", files)
c.Assert(err, IsNil)
c.Assert(len(files.Files), Equals, 2)
f := files.Files[1]
c.Assert(f.Name, Equals, "banana")
c.Assert(f.Size, Equals, uint64(100))
c.Assert(f.IsDir, Equals, false)
f = files.Files[0]
c.Assert(f.Name, Equals, "sampledir")
c.Assert(f.Size, Equals, uint64(0))
c.Assert(f.IsDir, Equals, true)
}