Exemplo n.º 1
0
func NewS3Session(auth *aws.Auth, region aws.Region) *s3.S3 {
	var s3Session *s3.S3
	cert := x509.Certificate{}
	// go's systemVerify panics with no verify options set
	// TODO: EVG-483
	if runtime.GOOS == "windows" {
		s3Session = s3.New(*auth, region)
		s3Session.ReadTimeout = S3ReadTimeout
		s3Session.WriteTimeout = S3WriteTimeout
		s3Session.ConnectTimeout = S3ConnectTimeout
		return s3Session
	}
	// no verify options so system root ca will be used
	_, err := cert.Verify(x509.VerifyOptions{})
	rootsError := x509.SystemRootsError{}
	if err != nil && err.Error() == rootsError.Error() {
		// create a Transport which includes our TLSConfig with InsecureSkipVerify
		// and client timeouts.
		tlsConfig := tls.Config{InsecureSkipVerify: true}
		tr := http.Transport{
			TLSClientConfig: &tlsConfig}
		// add the Transport to our http client
		client := &http.Client{Transport: &tr}
		s3Session = s3.New(*auth, region, client)
	} else {
		s3Session = s3.New(*auth, region)
	}
	s3Session.ReadTimeout = S3ReadTimeout
	s3Session.WriteTimeout = S3WriteTimeout
	s3Session.ConnectTimeout = S3ConnectTimeout
	return s3Session
}
Exemplo n.º 2
0
func NewS3Session(auth *aws.Auth, region aws.Region) *s3.S3 {
	if runtime.GOOS == "darwin" {
		// create a Transport which includes our TLS config
		tlsConfig := tls.Config{InsecureSkipVerify: true}
		tr := http.Transport{TLSClientConfig: &tlsConfig}
		// add the Transport to our http client
		client := &http.Client{Transport: &tr}
		return s3.New(*auth, region, client)
	}
	return s3.New(*auth, region)

}
Exemplo n.º 3
0
func ReadFile(path string) ([]byte, *model.AppError) {

	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
		var auth aws.Auth
		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
		auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey

		s := s3.New(auth, awsRegion())
		bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)

		// try to get the file from S3 with some basic retry logic
		tries := 0
		for {
			tries++

			f, err := bucket.Get(path)

			if f != nil {
				return f, nil
			} else if tries >= 3 {
				return nil, model.NewLocAppError("ReadFile", "api.file.read_file.get.app_error", nil, "path="+path+", err="+err.Error())
			}
			time.Sleep(3000 * time.Millisecond)
		}
	} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
		if f, err := ioutil.ReadFile(utils.Cfg.FileSettings.Directory + path); err != nil {
			return nil, model.NewLocAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error())
		} else {
			return f, nil
		}
	} else {
		return nil, model.NewLocAppError("ReadFile", "api.file.read_file.configured.app_error", nil, "")
	}
}
Exemplo n.º 4
0
func uploadImageToS3(path string, fileName string) error {
	fmt.Printf("Filename: %s\n", fileName)

	auth := aws.Auth{
		AccessKey: os.Getenv("ACCESS_KEY"),
		SecretKey: os.Getenv("SECRET_KEY"),
	}

	var region = aws.USEast

	client := s3.New(auth, region)

	data, err := ioutil.ReadFile(fileName)

	if err != nil {
		panic("error reading file! " + fileName)
	}

	bucket := client.Bucket("mesos-hackathon-bucket")
	options := s3.Options{}

	fmt.Printf("Path: %s\n", path)
	err = bucket.Put(path, data, "binary/octet-stream", s3.PublicRead, options)
	if err != nil {
		return err
	}

	return nil
}
Exemplo n.º 5
0
func writeFile(f []byte, path string) *model.AppError {

	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
		var auth aws.Auth
		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
		auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey

		s := s3.New(auth, awsRegion())
		bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)

		ext := filepath.Ext(path)

		var err error
		if model.IsFileExtImage(ext) {
			options := s3.Options{}
			err = bucket.Put(path, f, model.GetImageMimeType(ext), s3.Private, options)

		} else {
			options := s3.Options{}
			err = bucket.Put(path, f, "binary/octet-stream", s3.Private, options)
		}

		if err != nil {
			return model.NewAppError("writeFile", "Encountered an error writing to S3", err.Error())
		}
	} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
		if err := writeFileLocally(f, utils.Cfg.FileSettings.Directory+path); err != nil {
			return err
		}
	} else {
		return model.NewAppError("writeFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
	}

	return nil
}
Exemplo n.º 6
0
func TestPutGet(t *testing.T) {
	Reset(t)
	auth, err := aws.EnvAuth()

	if err != nil {
		t.Error(err)
	}

	s := s3.New(auth, localRegion)

	b := s.Bucket("TestBucket")

	err = b.PutBucket("acl")

	if err != nil {
		t.Fatal(err)
	}

	o, err := b.GetBucketContents()

	if err != nil {
		t.Fatal(err)
	}

	if len(*o) != 0 {
		t.Fatalf("Bucket should be empty, but has %d object", len(*o))
	}
}
Exemplo n.º 7
0
func moveFile(oldPath, newPath string) *model.AppError {
	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
		fileData := make(chan []byte)
		getFileAndForget(oldPath, fileData)
		fileBytes := <-fileData

		if fileBytes == nil {
			return model.NewLocAppError("moveFile", "api.file.move_file.get_from_s3.app_error", nil, "")
		}

		var auth aws.Auth
		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
		auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey

		s := s3.New(auth, awsRegion())
		bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)

		if err := bucket.Del(oldPath); err != nil {
			return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error())
		}

		if err := writeFile(fileBytes, newPath); err != nil {
			return err
		}
	} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
		if err := os.Rename(utils.Cfg.FileSettings.Directory+oldPath, utils.Cfg.FileSettings.Directory+newPath); err != nil {
			return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error())
		}
	} else {
		return model.NewLocAppError("moveFile", "api.file.move_file.configured.app_error", nil, "")
	}

	return nil
}
Exemplo n.º 8
0
func readFile(path string) ([]byte, *model.AppError) {

	if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
		var auth aws.Auth
		auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
		auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey

		s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
		bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)

		// try to get the file from S3 with some basic retry logic
		tries := 0
		for {
			tries++

			f, err := bucket.Get(path)

			if f != nil {
				return f, nil
			} else if tries >= 3 {
				return nil, model.NewAppError("readFile", "Unable to get file from S3", "path="+path+", err="+err.Error())
			}
			time.Sleep(3000 * time.Millisecond)
		}
	} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
		if f, err := ioutil.ReadFile(utils.Cfg.ServiceSettings.StorageDirectory + path); err != nil {
			return nil, model.NewAppError("readFile", "Encountered an error reading from local server storage", err.Error())
		} else {
			return f, nil
		}
	} else {
		return nil, model.NewAppError("readFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
	}
}
Exemplo n.º 9
0
func WriteFile(f []byte, path string) *model.AppError {

	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
		var auth aws.Auth
		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
		auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey

		s := s3.New(auth, awsRegion())
		bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)

		ext := filepath.Ext(path)

		var err error
		if model.IsFileExtImage(ext) {
			options := s3.Options{}
			err = bucket.Put(path, f, model.GetImageMimeType(ext), s3.Private, options)

		} else {
			options := s3.Options{}
			err = bucket.Put(path, f, "binary/octet-stream", s3.Private, options)
		}

		if err != nil {
			return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error())
		}
	} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
		if err := WriteFileLocally(f, utils.Cfg.FileSettings.Directory+path); err != nil {
			return err
		}
	} else {
		return model.NewLocAppError("WriteFile", "api.file.write_file.configured.app_error", nil, "")
	}

	return nil
}
Exemplo n.º 10
0
func (rc *Recommender) LoadBackup() (success bool) {
	log.Info("Loading backup from S3:", rc.identifier)
	auth, err := aws.EnvAuth()
	if err != nil {
		log.Error("Problem trying to connect with AWS:", err)
		return false
	}

	s := s3.New(auth, rc.s3Region)
	bucket := s.Bucket(S3BUCKET)

	jsonData, err := bucket.Get(rc.getS3Path())
	if err != nil {
		log.Info("Problem trying to get backup from S3:", err)
		return false
	}

	dataFromJSON := [][]uint64{}
	json.Unmarshal(rc.uncompress(jsonData), &dataFromJSON)

	log.Info("Data loaded from S3:", rc.identifier, "len:", len(dataFromJSON))
	recs := 0
	for _, record := range dataFromJSON {
		scores := make(map[uint64]uint8)
		for i := 1; i < len(record); i += 2 {
			scores[record[i]] = uint8(record[i+1])
		}
		recs += len(scores)
		rc.AddRecord(record[0], scores)
	}

	return true
}
Exemplo n.º 11
0
// Communicate with all endpoints to see if they are alive.
func (s *ClientTests) TestRegions(c *gocheck.C) {
	errs := make(chan error, len(aws.Regions))
	for _, region := range aws.Regions {
		go func(r aws.Region) {
			s := s3.New(s.s3.Auth, r)
			b := s.Bucket("goamz-" + s.Auth.AccessKey)
			_, err := b.Get("non-existent")
			errs <- err
		}(region)
	}
	for _ = range aws.Regions {
		err := <-errs
		if err != nil {
			s3_err, ok := err.(*s3.Error)
			if ok {
				c.Check(s3_err.Code, gocheck.Matches, "NoSuchBucket")
			} else if _, ok = err.(*net.DNSError); ok {
				// Okay as well.
			} else {
				c.Errorf("Non-S3 error: %s", err)
			}
		} else {
			c.Errorf("Test should have errored but it seems to have succeeded")
		}
	}
}
Exemplo n.º 12
0
func readFile(path string) ([]byte, *model.AppError) {

	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
		var auth aws.Auth
		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
		auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey

		s := s3.New(auth, awsRegion())
		bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)

		// try to get the file from S3 with some basic retry logic
		tries := 0
		for {
			tries++

			f, err := bucket.Get(path)

			if f != nil {
				return f, nil
			} else if tries >= 3 {
				return nil, model.NewAppError("readFile", "Unable to get file from S3", "path="+path+", err="+err.Error())
			}
			time.Sleep(3000 * time.Millisecond)
		}
	} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
		if f, err := ioutil.ReadFile(utils.Cfg.FileSettings.Directory + path); err != nil {
			return nil, model.NewAppError("readFile", "Encountered an error reading from local server storage", err.Error())
		} else {
			return f, nil
		}
	} else {
		return nil, model.NewAppError("readFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
	}
}
func goamzBucket(bucketName, endpoint string) *goamz.Bucket {
	region := aws.Region{
		Name:                 "fake_region",
		S3Endpoint:           s3Server.URL(),
		S3LocationConstraint: true,
	}
	return goamz.New(aws.Auth{}, region).Bucket(bucketName)
}
Exemplo n.º 14
0
func (s3p *S3Provider) Version(rawurl, previous string) (string, error) {
	ref := s3p.parse(rawurl)
	resp, err := s3.New(ref.auth, ref.region).Bucket(ref.bucket).Head(ref.path, map[string][]string{})
	if err != nil {
		return "", err
	}
	return resp.Header.Get("x-amz-version-id"), nil
}
Exemplo n.º 15
0
func (s *LocalServerSuite) SetUpSuite(c *C) {
	s.srv.SetUp(c)
	s.clientTests.s3 = s3.New(s.srv.auth, s.srv.region)

	// TODO Sadly the fake server ignores auth completely right now. :-(
	s.clientTests.authIsBroken = true
	s.clientTests.Cleanup()
}
Exemplo n.º 16
0
func s3Client() *s3.S3 {
	r := aws.Region{
		Name:       "jp-east",
		S3Endpoint: "https://ds.jp-east.idcfcloud.com",
	}
	auth := aws.NewAuth(os.Getenv("IDCF_ACCESS_KEY"), os.Getenv("IDCF_ACCESS_SECRET"), "", time.Now())
	return s3.New(*auth, r)
}
Exemplo n.º 17
0
func GetS3File(auth *aws.Auth, s3URL string) (io.ReadCloser, error) {
	urlParsed, err := url.Parse(s3URL)
	if err != nil {
		return nil, err
	}
	session := s3.New(*auth, aws.USEast)
	bucket := session.Bucket(urlParsed.Host)
	return bucket.GetReader(urlParsed.Path)
}
Exemplo n.º 18
0
func s3BucketFromConfig(c Config) *s3.Bucket {
	awsAuth := aws.Auth{
		AccessKey: c.AwsClientKey,
		SecretKey: c.AwsSecretKey,
	}
	region := aws.USEast
	connection := s3.New(awsAuth, region)
	return connection.Bucket(c.Bucket)
}
Exemplo n.º 19
0
func (s *AmazonClientSuite) SetUpSuite(c *gocheck.C) {
	if !testutil.Amazon {
		c.Skip("live tests against AWS disabled (no -amazon)")
	}
	s.srv.SetUp(c)
	s.s3 = s3.New(s.srv.auth, s.Region)
	// In case tests were interrupted in the middle before.
	s.ClientTests.Cleanup()
}
Exemplo n.º 20
0
// NewS3Client returns a new S3 client
func (m *AwsMgr) NewS3Client(accId, region string) (*s3.S3, error) {
	// Get Auth
	auth, err := m.GetAuth(accId)
	if err != nil {
		return nil, err
	}
	s3 := s3.New(*auth, aws.Regions[region])
	return s3, nil
}
Exemplo n.º 21
0
func getBucket() (*s3.Bucket, error) {
	auth, err := aws.GetAuth(config.AWS_ACCESS_KEY, config.AWS_SECRET_KEY, "", time.Time{})
	if err != nil {
		return nil, err
	}

	conn := s3.New(auth, aws.Regions["eu-west-1"])
	b := conn.Bucket(config.BUCKET)
	return b, nil
}
Exemplo n.º 22
0
func (s *AmazonDomainClientSuite) SetUpSuite(c *gocheck.C) {
	if !testutil.Amazon {
		c.Skip("live tests against AWS disabled (no -amazon)")
	}
	s.srv.SetUp(c)
	region := s.Region
	region.S3BucketEndpoint = "https://${bucket}.s3.amazonaws.com"
	s.s3 = s3.New(s.srv.auth, region)
	s.ClientTests.Cleanup()
}
Exemplo n.º 23
0
func (s3w S3Writer) WriteToS3(p []byte) error {
	auth := aws.Auth{
		AccessKey: s3w.AccessKey,
		SecretKey: s3w.SecretKey,
	}
	path := fmt.Sprintf("%s-%s.tar.gz", s3w.ClusterName, time.Now().UTC().Format(time.RFC3339))

	client := s3.New(auth, aws.Region{S3Endpoint: s3w.Endpoint})
	bucket := client.Bucket(s3w.Bucket)
	return bucket.Put(path, p, "application/x-gzip", s3.Private, s3.Options{})
}
Exemplo n.º 24
0
func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	id := params["id"]

	if result := <-Srv.Store.User().Get(id); result.Err != nil {
		c.Err = result.Err
		return
	} else {
		var img []byte
		var err *model.AppError

		if !utils.IsS3Configured() {
			img, err = createProfileImage(result.Data.(*model.User).Username, id)
			if err != nil {
				c.Err = err
				return
			}
		} else {
			var auth aws.Auth
			auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
			auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey

			s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
			bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)

			path := "teams/" + c.Session.TeamId + "/users/" + id + "/profile.png"

			if data, getErr := bucket.Get(path); getErr != nil {
				img, err = createProfileImage(result.Data.(*model.User).Username, id)
				if err != nil {
					c.Err = err
					return
				}

				options := s3.Options{}
				if err := bucket.Put(path, img, "image", s3.Private, options); err != nil {
					c.Err = model.NewAppError("getImage", "Couldn't upload default profile image", err.Error())
					return
				}

			} else {
				img = data
			}
		}

		if c.Session.UserId == id {
			w.Header().Set("Cache-Control", "max-age=300, public") // 5 mins
		} else {
			w.Header().Set("Cache-Control", "max-age=86400, public") // 24 hrs
		}

		w.Write(img)
	}
}
Exemplo n.º 25
0
Arquivo: s3.go Projeto: aya/pb
func (s *S3MultipartUploadSession) getS3Bucket() *s3.Bucket {
	auth := s.awsAuth()
	s3 := s3.New(auth, aws.Regions[s.s3Region])

	s3.ConnectTimeout = time.Second * 10
	s3.ReadTimeout = time.Second * 20
	s3.WriteTimeout = time.Second * 20
	s3.RequestTimeout = time.Second * 120

	return s3.Bucket(s.s3Bucket)
}
Exemplo n.º 26
0
func s3GetBucket() *s3.Bucket {
	s3VerifyHasSecrets()
	auth := aws.Auth{
		AccessKey: s3AwsAccess,
		SecretKey: s3AwsSecret,
	}
	// Note: it's important that region is aws.USEast. This is where my bucket
	// is and giving a different region will fail
	// TODO: make aws.USEast a variable s3BucketRegion, to allow over-ride
	s3Obj := s3.New(auth, aws.USEast, getS3Client())
	return s3Obj.Bucket(s3BucketName)
}
Exemplo n.º 27
0
func NewClient(endpoint, accessKey, secretKey string, logger lager.Logger) Client {
	auth := aws.Auth{
		AccessKey: accessKey,
		SecretKey: secretKey,
	}

	return &s3Client{
		endpoint:    endpoint,
		goamzClient: goamz.New(auth, getRegion(endpoint)),
		logger:      logger,
	}
}
Exemplo n.º 28
0
// Getbucket connects to bucket name, with connection c, and returns
// a bucket type and any connection error
func Getbucket(name string, c *Connection) (*s3.Bucket, error) {
	s3Connector := s3.New(c.Auth, c.Region)
	bucket := s3Connector.Bucket(name)
	_, err := bucket.List("", "", "", 1)
	if err != nil {
		// probably wrong name of bucket
		log.Errorf("connection to s3", name, err)
	} else {
		fmt.Printf("Connected to s3 bucket: %v\n", name)
	}
	return bucket, err
}
Exemplo n.º 29
0
func s3GetBucket() *s3.Bucket {
	s3BucketName := "kjkpub"
	secrets := readSecretsMust()
	auth := aws.Auth{
		AccessKey: secrets.AwsAccess,
		SecretKey: secrets.AwsSecret,
	}
	// Note: it's important that region is aws.USEast. This is where my bucket
	// is and giving a different region will fail
	s3Obj := s3.New(auth, aws.USEast, getS3Client())
	return s3Obj.Bucket(s3BucketName)
}
Exemplo n.º 30
0
/**
 This func we need to initialize our backup service in main programm
**/
func NewBackup(db *DBConf, awsc *Aws, backupdir string, rlimit, cTimeout, wTimeout, rTimeout int) *Backup {
	quit := make(chan struct{})
	started := make(chan struct{})
	err := make(chan error)

	zone := aws.EUWest
	connection := s3.New(awsc.Auth, zone)
	connection.ConnectTimeout = time.Duration(cTimeout) * time.Second
	connection.WriteTimeout = time.Duration(wTimeout) * time.Second
	connection.RequestTimeout = time.Duration(rTimeout) * time.Second

	return &Backup{db, awsc, backupdir, rlimit, connection, quit, started, err}
}