Ejemplo n.º 1
0
func AssertOnMultipartUploads(s3CLIPath string, cfg *config.S3Cli, content string) {
	s3Filename := GenerateRandomString()
	sourceContent := strings.NewReader(content)

	configPath := MakeConfigFile(cfg)
	defer func() { _ = os.Remove(configPath) }()

	configFile, err := os.Open(configPath)
	Expect(err).ToNot(HaveOccurred())

	s3Config, err := config.NewFromReader(configFile)
	Expect(err).ToNot(HaveOccurred())

	s3Client, err := client.NewSDK(s3Config)
	if err != nil {
		log.Fatalln(err)
	}

	tracedS3, calls := traceS3(s3Client)

	blobstoreClient, err := client.New(tracedS3, &s3Config)
	Expect(err).ToNot(HaveOccurred())

	err = blobstoreClient.Put(sourceContent, s3Filename)
	Expect(err).ToNot(HaveOccurred())

	switch cfg.Host {
	case "storage.googleapis.com":
		Expect(*calls).To(Equal([]string{"PutObject"}))
	default:
		Expect(*calls).To(Equal([]string{"CreateMultipart", "UploadPart", "UploadPart", "CompleteMultipart"}))
	}
}
Ejemplo n.º 2
0
// AssertPutOptionsApplied asserts that `s3cli put` uploads files with
// the requested encryption options
func AssertPutOptionsApplied(s3CLIPath string, cfg *config.S3Cli) {
	expectedString := GenerateRandomString()
	s3Filename := GenerateRandomString()

	configPath := MakeConfigFile(cfg)
	defer func() { _ = os.Remove(configPath) }()

	contentFile := MakeContentFile(expectedString)
	defer func() { _ = os.Remove(contentFile) }()

	configFile, err := os.Open(configPath)
	Expect(err).ToNot(HaveOccurred())

	s3CLISession, err := RunS3CLI(s3CLIPath, configPath, "put", contentFile, s3Filename)

	s3Config, err := config.NewFromReader(configFile)
	Expect(err).ToNot(HaveOccurred())

	client, _ := client.NewSDK(s3Config)
	resp, err := client.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(cfg.BucketName),
		Key:    aws.String(s3Filename),
	})
	Expect(err).ToNot(HaveOccurred())
	Expect(s3CLISession.ExitCode()).To(BeZero())

	if cfg.ServerSideEncryption == "" {
		Expect(resp.ServerSideEncryption).To(BeNil())
	} else {
		Expect(*resp.ServerSideEncryption).To(Equal(cfg.ServerSideEncryption))
	}
}
Ejemplo n.º 3
0
func AssertOnPutFailures(s3CLIPath string, cfg *config.S3Cli, content, errorMessage string) {
	s3Filename := GenerateRandomString()
	sourceContent := strings.NewReader(content)

	configPath := MakeConfigFile(cfg)
	defer func() { _ = os.Remove(configPath) }()

	configFile, err := os.Open(configPath)
	Expect(err).ToNot(HaveOccurred())

	s3Config, err := config.NewFromReader(configFile)
	Expect(err).ToNot(HaveOccurred())

	s3Client, err := client.NewSDK(s3Config)
	if err != nil {
		log.Fatalln(err)
	}

	// Here we cause every other "part" to fail by way of SHA mismatch,
	// thereby triggering the MultiUploadFailure which causes retries.
	part := 0
	s3Client.Handlers.Build.PushBack(func(r *request.Request) {
		if r.Operation.Name == "UploadPart" {
			if part%2 == 0 {
				r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "000")
			}
			part++
		}
	})

	blobstoreClient, err := client.New(s3Client, &s3Config)
	Expect(err).ToNot(HaveOccurred())

	err = blobstoreClient.Put(sourceContent, s3Filename)
	Expect(err).To(HaveOccurred())
	Expect(err.Error()).To(ContainSubstring(errorMessage))
}
Ejemplo n.º 4
0
		It("allows for the S3 SDK to be configured with empty region information", func() {
			Expect(config.EmptyRegion).To(Equal(" "))
		})
	})

	Describe("building a configuration", func() {
		Describe("checking that either host or region has been set", func() {

			Context("when AWS endpoint has been set but not region", func() {

				It("sets the AWS region based on the hostname", func() {
					Expect(config.AWSHostToRegion).ToNot(BeEmpty())
					for endpoint, region := range config.AWSHostToRegion {
						dummyJSONBytes := []byte(fmt.Sprintf(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "%s"}`, endpoint))
						dummyJSONReader := bytes.NewReader(dummyJSONBytes)
						c, err := config.NewFromReader(dummyJSONReader)
						Expect(err).ToNot(HaveOccurred())
						Expect(c.UseRegion()).To(BeTrue(), "Expected UseRegion to be true")
						Expect(c.Host).To(Equal(endpoint))
						Expect(c.Region).To(Equal(region))
					}
				})
			})

			Context("when non-AWS endpoint has been set but not region", func() {
				dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "some-host"}`)
				dummyJSONReader := bytes.NewReader(dummyJSONBytes)

				It("reports that region should not be used for SDK configuration", func() {
					c, err := config.NewFromReader(dummyJSONReader)
					Expect(err).ToNot(HaveOccurred())
Ejemplo n.º 5
0
func main() {
	configPath := flag.String("c", "", "configuration path")
	showVer := flag.Bool("v", false, "version")
	flag.Parse()

	if *showVer {
		fmt.Printf("version %s\n", version)
		os.Exit(0)
	}

	configFile, err := os.Open(*configPath)
	if err != nil {
		log.Fatalln(err)
	}

	s3Config, err := config.NewFromReader(configFile)
	if err != nil {
		log.Fatalln(err)
	}

	s3Client, err := client.NewSDK(s3Config)
	if err != nil {
		log.Fatalln(err)
	}

	blobstoreClient, err := client.New(s3Client, &s3Config)
	if err != nil {
		log.Fatalln(err)
	}

	nonFlagArgs := flag.Args()
	if len(nonFlagArgs) < 2 {
		log.Fatalf("Expected at least two arguments got %d\n", len(nonFlagArgs))
	}

	cmd := nonFlagArgs[0]

	switch cmd {
	case "put":
		if len(nonFlagArgs) != 3 {
			log.Fatalf("Put method expected 3 arguments got %d\n", len(nonFlagArgs))
		}
		src, dst := nonFlagArgs[1], nonFlagArgs[2]

		var sourceFile *os.File
		sourceFile, err = os.Open(src)
		if err != nil {
			log.Fatalln(err)
		}

		defer sourceFile.Close()
		err = blobstoreClient.Put(sourceFile, dst)
	case "get":
		if len(nonFlagArgs) != 3 {
			log.Fatalf("Get method expected 3 arguments got %d\n", len(nonFlagArgs))
		}
		src, dst := nonFlagArgs[1], nonFlagArgs[2]

		var dstFile *os.File
		dstFile, err = os.Create(dst)
		if err != nil {
			log.Fatalln(err)
		}

		defer dstFile.Close()
		err = blobstoreClient.Get(src, dstFile)
	case "delete":
		if len(nonFlagArgs) != 2 {
			log.Fatalf("Delete method expected 2 arguments got %d\n", len(nonFlagArgs))
		}

		err = blobstoreClient.Delete(nonFlagArgs[1])
	case "exists":
		if len(nonFlagArgs) != 2 {
			log.Fatalf("Exists method expected 2 arguments got %d\n", len(nonFlagArgs))
		}

		var exists bool
		exists, err = blobstoreClient.Exists(nonFlagArgs[1])

		// If the object exists the exit status is 0, otherwise it is 3
		// We are using `3` since `1` and `2` have special meanings
		if err == nil && !exists {
			os.Exit(3)
		}

	default:
		log.Fatalf("unknown command: '%s'\n", cmd)
	}

	if err != nil {
		log.Fatalf("performing operation %s: %s\n", cmd, err)
	}
}