func AssertOnMultipartUploads(s3CLIPath string, cfg *config.S3Cli, content string) {
s3Filename := GenerateRandomString()
sourceContent := strings.NewReader(content)
configPath := MakeConfigFile(cfg)
defer func() { _ = os.Remove(configPath) }()
configFile, err := os.Open(configPath)
Expect(err).ToNot(HaveOccurred())
s3Config, err := config.NewFromReader(configFile)
Expect(err).ToNot(HaveOccurred())
s3Client, err := client.NewSDK(s3Config)
if err != nil {
log.Fatalln(err)
}
tracedS3, calls := traceS3(s3Client)
blobstoreClient, err := client.New(tracedS3, &s3Config)
Expect(err).ToNot(HaveOccurred())
err = blobstoreClient.Put(sourceContent, s3Filename)
Expect(err).ToNot(HaveOccurred())
switch cfg.Host {
case "storage.googleapis.com":
Expect(*calls).To(Equal([]string{"PutObject"}))
default:
Expect(*calls).To(Equal([]string{"CreateMultipart", "UploadPart", "UploadPart", "CompleteMultipart"}))
}
}
// AssertPutOptionsApplied asserts that `s3cli put` uploads files with
// the requested encryption options
func AssertPutOptionsApplied(s3CLIPath string, cfg *config.S3Cli) {
expectedString := GenerateRandomString()
s3Filename := GenerateRandomString()
configPath := MakeConfigFile(cfg)
defer func() { _ = os.Remove(configPath) }()
contentFile := MakeContentFile(expectedString)
defer func() { _ = os.Remove(contentFile) }()
configFile, err := os.Open(configPath)
Expect(err).ToNot(HaveOccurred())
s3CLISession, err := RunS3CLI(s3CLIPath, configPath, "put", contentFile, s3Filename)
s3Config, err := config.NewFromReader(configFile)
Expect(err).ToNot(HaveOccurred())
client, _ := client.NewSDK(s3Config)
resp, err := client.HeadObject(&s3.HeadObjectInput{
Bucket: aws.String(cfg.BucketName),
Key: aws.String(s3Filename),
})
Expect(err).ToNot(HaveOccurred())
Expect(s3CLISession.ExitCode()).To(BeZero())
if cfg.ServerSideEncryption == "" {
Expect(resp.ServerSideEncryption).To(BeNil())
} else {
Expect(*resp.ServerSideEncryption).To(Equal(cfg.ServerSideEncryption))
}
}
func AssertOnPutFailures(s3CLIPath string, cfg *config.S3Cli, content, errorMessage string) {
s3Filename := GenerateRandomString()
sourceContent := strings.NewReader(content)
configPath := MakeConfigFile(cfg)
defer func() { _ = os.Remove(configPath) }()
configFile, err := os.Open(configPath)
Expect(err).ToNot(HaveOccurred())
s3Config, err := config.NewFromReader(configFile)
Expect(err).ToNot(HaveOccurred())
s3Client, err := client.NewSDK(s3Config)
if err != nil {
log.Fatalln(err)
}
// Here we cause every other "part" to fail by way of SHA mismatch,
// thereby triggering the MultiUploadFailure which causes retries.
part := 0
s3Client.Handlers.Build.PushBack(func(r *request.Request) {
if r.Operation.Name == "UploadPart" {
if part%2 == 0 {
r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "000")
}
part++
}
})
blobstoreClient, err := client.New(s3Client, &s3Config)
Expect(err).ToNot(HaveOccurred())
err = blobstoreClient.Put(sourceContent, s3Filename)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(errorMessage))
}
It("allows for the S3 SDK to be configured with empty region information", func() {
Expect(config.EmptyRegion).To(Equal(" "))
})
})
Describe("building a configuration", func() {
Describe("checking that either host or region has been set", func() {
Context("when AWS endpoint has been set but not region", func() {
It("sets the AWS region based on the hostname", func() {
Expect(config.AWSHostToRegion).ToNot(BeEmpty())
for endpoint, region := range config.AWSHostToRegion {
dummyJSONBytes := []byte(fmt.Sprintf(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "%s"}`, endpoint))
dummyJSONReader := bytes.NewReader(dummyJSONBytes)
c, err := config.NewFromReader(dummyJSONReader)
Expect(err).ToNot(HaveOccurred())
Expect(c.UseRegion()).To(BeTrue(), "Expected UseRegion to be true")
Expect(c.Host).To(Equal(endpoint))
Expect(c.Region).To(Equal(region))
}
})
})
Context("when non-AWS endpoint has been set but not region", func() {
dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "some-host"}`)
dummyJSONReader := bytes.NewReader(dummyJSONBytes)
It("reports that region should not be used for SDK configuration", func() {
c, err := config.NewFromReader(dummyJSONReader)
Expect(err).ToNot(HaveOccurred())
func main() {
configPath := flag.String("c", "", "configuration path")
showVer := flag.Bool("v", false, "version")
flag.Parse()
if *showVer {
fmt.Printf("version %s\n", version)
os.Exit(0)
}
configFile, err := os.Open(*configPath)
if err != nil {
log.Fatalln(err)
}
s3Config, err := config.NewFromReader(configFile)
if err != nil {
log.Fatalln(err)
}
s3Client, err := client.NewSDK(s3Config)
if err != nil {
log.Fatalln(err)
}
blobstoreClient, err := client.New(s3Client, &s3Config)
if err != nil {
log.Fatalln(err)
}
nonFlagArgs := flag.Args()
if len(nonFlagArgs) < 2 {
log.Fatalf("Expected at least two arguments got %d\n", len(nonFlagArgs))
}
cmd := nonFlagArgs[0]
switch cmd {
case "put":
if len(nonFlagArgs) != 3 {
log.Fatalf("Put method expected 3 arguments got %d\n", len(nonFlagArgs))
}
src, dst := nonFlagArgs[1], nonFlagArgs[2]
var sourceFile *os.File
sourceFile, err = os.Open(src)
if err != nil {
log.Fatalln(err)
}
defer sourceFile.Close()
err = blobstoreClient.Put(sourceFile, dst)
case "get":
if len(nonFlagArgs) != 3 {
log.Fatalf("Get method expected 3 arguments got %d\n", len(nonFlagArgs))
}
src, dst := nonFlagArgs[1], nonFlagArgs[2]
var dstFile *os.File
dstFile, err = os.Create(dst)
if err != nil {
log.Fatalln(err)
}
defer dstFile.Close()
err = blobstoreClient.Get(src, dstFile)
case "delete":
if len(nonFlagArgs) != 2 {
log.Fatalf("Delete method expected 2 arguments got %d\n", len(nonFlagArgs))
}
err = blobstoreClient.Delete(nonFlagArgs[1])
case "exists":
if len(nonFlagArgs) != 2 {
log.Fatalf("Exists method expected 2 arguments got %d\n", len(nonFlagArgs))
}
var exists bool
exists, err = blobstoreClient.Exists(nonFlagArgs[1])
// If the object exists the exit status is 0, otherwise it is 3
// We are using `3` since `1` and `2` have special meanings
if err == nil && !exists {
os.Exit(3)
}
default:
log.Fatalf("unknown command: '%s'\n", cmd)
}
if err != nil {
log.Fatalf("performing operation %s: %s\n", cmd, err)
}
}