func (s *S3) Download(remote_path, local_path string, uid, gid int, perms, listener string) error {
log.Infof("S3 Downloading %s", local_path)
conf, keys := s.GetS3Config()
// Open bucket to put file into
s3 := s3gof3r.New("", *keys)
b := s3.Bucket(s.config.Listeners[s.listener].Bucket)
r, _, err := b.GetReader(remote_path, conf)
if err != nil {
return err
}
// stream to file
if _, err = utils.FileWrite(local_path, r, uid, gid, perms); err != nil {
return err
}
err = r.Close()
if err != nil {
return err
}
basePath := s.config.Listeners[listener].BasePath
datastore.UpdateHost(listener, strings.TrimLeft(local_path, basePath))
return err
}
func main() {
s3gof3r.SetLogger(os.Stdout, "", log.LstdFlags, false)
// STARTEXAMPLE OMIT
presentation, err := os.Open(file_name) // open presentation file
if err != nil {
log.Fatal(err)
}
k, err := s3gof3r.EnvKeys() // get S3 keys from environment
if err != nil {
log.Fatal(err)
}
// Open bucket to put file into
s3 := s3gof3r.New("", k)
b := s3.Bucket(bucket_name)
// Open a PutWriter for upload
w, err := b.PutWriter(presentation.Name(), nil, nil)
if err != nil {
log.Fatal(err)
}
defer w.Close()
if _, err = io.Copy(w, presentation); err != nil { // Copy into S3
log.Fatal(err)
}
log.Printf("%s uploaded to %s", file_name, bucket_name)
// STOPEXAMPLE OMIT
}
func reloc(bucketName, oldPath, newPath string, keys s3gof3r.Keys) {
s3 := s3gof3r.New("s3.amazonaws.com", keys)
bucket := s3.Bucket(bucketName)
// this is a POST at the bottom, and copies are a PUT. whee.
//w, err := s3.Bucket(bucketName).PutWriter(newPath, copyInstruction, s3Conf)
// So, implement our own aws copy API.
req, err := http.NewRequest("PUT", "", &bytes.Buffer{})
if err != nil {
panic(integrity.WarehouseConnectionError.Wrap(err))
}
req.URL.Scheme = s3Conf.Scheme
req.URL.Host = fmt.Sprintf("%s.%s", bucketName, s3.Domain)
req.URL.Path = path.Clean(fmt.Sprintf("/%s", newPath))
// Communicate the copy source object with a header.
// Be advised that if this object doesn't exist, amazon reports that as a 404... yes, a 404 that has nothing to do with the query URI.
req.Header.Add("x-amz-copy-source", path.Join("/", bucketName, oldPath))
bucket.Sign(req)
resp, err := s3Conf.Client.Do(req)
if err != nil {
panic(integrity.WarehouseConnectionError.Wrap(err))
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
panic(integrity.WarehouseConnectionError.Wrap(newRespError(resp)))
}
// delete previous location
if err := bucket.Delete(oldPath); err != nil {
panic(integrity.WarehouseConnectionError.Wrap(err))
}
}
func (s *S3) Upload(local_path, remote_path string) error {
conf, keys := s.GetS3Config()
// Open bucket to put file into
s3 := s3gof3r.New("", *keys)
b := s3.Bucket(s.config.Listeners[s.listener].Bucket)
// open file to upload
file, err := os.Open(local_path)
if err != nil {
return err
}
// Open a PutWriter for upload
w, err := b.PutWriter(remote_path, nil, conf)
if err != nil {
return err
}
if _, err = io.Copy(w, file); err != nil { // Copy into S3
return err
}
if err = w.Close(); err != nil {
return err
}
return nil
}
func DownloadFromS3(k s3gof3r.Keys, bucketName, filename string) (bytes.Buffer, error) {
s3 := s3gof3r.New("", k)
b := s3.Bucket(bucketName)
r, _, err := b.GetReader(filename, nil)
if err != nil {
return bytes.Buffer{}, err
}
// outputFile, err := os.Create(filepath.Join(outputDirPath, filename))
// if err != nil {
// return nil, err
// }
var output bytes.Buffer
if _, err = io.Copy(&output, r); err != nil {
return bytes.Buffer{}, err
}
err = r.Close()
if err != nil {
return bytes.Buffer{}, err
}
return output, nil
}
func (rm *rmOpts) Execute(args []string) error {
k, err := getAWSKeys()
if err != nil {
return err
}
conf := new(s3gof3r.Config)
*conf = *s3gof3r.DefaultConfig
s3 := s3gof3r.New(rm.EndPoint, k)
s3gof3r.SetLogger(os.Stderr, "", log.Ltime, rm.Debug)
// parse positional cp args
if len(args) != 1 {
return fmt.Errorf("rm: path argument required")
}
//var urls [1]*url.URL
u, err := url.ParseRequestURI(args[0])
if err != nil {
return fmt.Errorf("parse error: %s", err)
}
if u.Host != "" && u.Scheme != "s3" {
return fmt.Errorf("parse error: %s", u.String())
}
return s3.Bucket(u.Host).Delete(u.Path)
}
func UploadToS3(k s3gof3r.Keys, bucketName string, pathToFile string) error {
s3 := s3gof3r.New("", k)
b := s3.Bucket(bucketName)
file, err := os.Open(pathToFile)
if err != nil {
return err
}
stats, _ := file.Stat()
dstPath := FormatDstPath("/backup/%hostname%/%timestamp%", stats.Name())
w, err := b.PutWriter(dstPath, nil, nil)
if err != nil {
return err
}
if _, err = io.Copy(w, file); err != nil { // Copy into S3
return err
}
if err = w.Close(); err != nil {
return err
}
return nil
}
//SafeCreateS3Bucket creates an s3 bucket for storing files to an s3-compatible blobstore
func SafeCreateS3Bucket(domain, bucket, accessKey, secretKey string) (*S3Bucket, error) {
s := &S3Bucket{
Bucket: bucket,
Name: "s3",
Domain: domain,
AccessKey: accessKey,
SecretKey: secretKey,
}
if s.Bucket == "" {
return nil, errors.New("bucket name is undefined")
}
var k s3gof3r.Keys
var err error
if s.AccessKey == "" || s.SecretKey == "" {
k, err = s3gof3r.EnvKeys() // get S3 keys from environment
if err != nil {
return nil, err
}
} else {
k = s3gof3r.Keys{
AccessKey: s.AccessKey,
SecretKey: s.SecretKey,
}
}
s3 := s3gof3r.New(s.Domain, k)
s.bucket = s3.Bucket(s.Bucket)
return s, nil
}
func makeS3writer(bucketName string, path string, keys s3gof3r.Keys) io.WriteCloser {
s3 := s3gof3r.New("s3.amazonaws.com", keys)
w, err := s3.Bucket(bucketName).PutWriter(path, nil, s3Conf)
if err != nil {
panic(integrity.WarehouseConnectionError.Wrap(err))
}
return w
}
func (cp *cpOpts) Execute(args []string) (err error) {
k, err := getAWSKeys()
if err != nil {
return
}
conf := new(s3gof3r.Config)
*conf = *s3gof3r.DefaultConfig
s3 := s3gof3r.New(cp.EndPoint, k)
conf.Concurrency = cp.Concurrency
if cp.NoSSL {
conf.Scheme = "http"
}
conf.PartSize = cp.PartSize
conf.Md5Check = !cp.NoMd5
s3gof3r.SetLogger(os.Stderr, "", log.LstdFlags, cp.Debug)
src, err := func(src string) (io.ReadCloser, error) {
u, err := url.Parse(src)
if err != nil {
return nil, fmt.Errorf("parse error: %s", err)
}
if u.Host == "" {
return os.Open(u.Path)
}
r, _, err := s3.Bucket(u.Host).GetReader(u.Path, conf)
return r, err
}(cp.Source)
if err != nil {
return
}
dst, err := func(dst string) (io.WriteCloser, error) {
u, err := url.Parse(dst)
if err != nil {
return nil, fmt.Errorf("parse error: %s", err)
}
if u.Host == "" {
return os.Create(u.Path)
}
return s3.Bucket(u.Host).PutWriter(u.Path, ACL(cp.Header, cp.ACL), conf)
}(cp.Dest)
if err != nil {
return
}
if _, err = io.Copy(dst, src); err != nil {
return
}
if err = src.Close(); err != nil {
return
}
return dst.Close()
}
func (s *S3) Remove(remote_path string) bool {
log.Infof("Removing file %s from s3 storage", remote_path)
_, keys := s.GetS3Config()
// Open bucket to put file into
s3 := s3gof3r.New("", *keys)
b := s3.Bucket(s.config.Listeners[s.listener].Bucket)
b.Delete(remote_path)
return true
}
func (cp *cpOpts) Execute(args []string) (err error) {
k, err := getAWSKeys()
if err != nil {
return
}
conf := new(s3gof3r.Config)
*conf = *s3gof3r.DefaultConfig
s3 := s3gof3r.New(cp.EndPoint, k)
conf.Concurrency = cp.Concurrency
if cp.NoSSL {
conf.Scheme = "http"
}
conf.PartSize = cp.PartSize
conf.Md5Check = !cp.NoMd5
conf.NTry = cp.NTry
s3gof3r.SetLogger(os.Stderr, "", log.LstdFlags, cp.Debug)
src, err := func(src string) (io.ReadCloser, error) {
if !strings.HasPrefix(strings.ToLower(src), "s3") {
return os.Open(src)
}
u, err := url.ParseRequestURI(src)
if err != nil {
return nil, fmt.Errorf("parse error: %s", err)
}
r, _, err := s3.Bucket(u.Host).GetReader(u.Path, conf)
return r, err
}(cp.Source)
if err != nil {
return
}
defer checkClose(src, err)
dst, err := func(dst string) (io.WriteCloser, error) {
if !strings.HasPrefix(strings.ToLower(dst), "s3") {
return os.Create(dst)
}
u, err := url.ParseRequestURI(dst)
if err != nil {
return nil, fmt.Errorf("parse error: %s", err)
}
return s3.Bucket(u.Host).PutWriter(u.Path, ACL(cp.Header, cp.ACL), conf)
}(cp.Dest)
if err != nil {
return
}
defer checkClose(dst, err)
_, err = io.Copy(dst, src)
return
}
func makeS3reader(bucketName string, path string, keys s3gof3r.Keys) io.ReadCloser {
s3 := s3gof3r.New("s3.amazonaws.com", keys)
w, _, err := s3.Bucket(bucketName).GetReader(path, s3Conf)
if err != nil {
if err2, ok := err.(*s3gof3r.RespError); ok && err2.Code == "NoSuchKey" {
panic(integrity.DataDNE.New("not stored here"))
} else {
panic(integrity.WarehouseConnectionError.Wrap(err))
}
}
return w
}
func newS3WriterProvider(awsAccessKey string, awsSecretKey string, s3Domain string, bucketName string) *S3WriterProvider {
s3gof3r.DefaultDomain = s3Domain
awsKeys := s3gof3r.Keys{
AccessKey: awsAccessKey,
SecretKey: awsSecretKey,
}
s3 := s3gof3r.New("", awsKeys)
bucket := s3.Bucket(bucketName)
return &S3WriterProvider{bucket}
}
func NewS3Remote(config config.Config) (*S3Remote, error) {
s3, err := newS3Client(config)
if err != nil {
return &S3Remote{}, err
}
s3gof3rKeys := s3gof3r.Keys{AccessKey: config.AWS.AccessKeyID, SecretKey: config.AWS.SecretAccessKey}
return &S3Remote{
config: config,
BucketName: config.AWS.S3URL.Host,
client: s3,
uploadDownloadClient: s3gof3r.New("", s3gof3rKeys),
}, nil
}
func (get *Get) Execute(args []string) (err error) {
conf := new(s3gof3r.Config)
*conf = *s3gof3r.DefaultConfig
k, err := getAWSKeys()
if err != nil {
return
}
s3 := s3gof3r.New(get.EndPoint, k)
b := s3.Bucket(get.Bucket)
if get.Concurrency > 0 {
conf.Concurrency = get.Concurrency
}
conf.PartSize = get.PartSize
conf.Md5Check = !get.CheckDisable
get.Key = url.QueryEscape(get.Key)
if get.VersionId != "" {
get.Key = fmt.Sprintf("%s?versionId=%s", get.Key, get.VersionId)
}
log.Println("GET: ", get)
w, err := os.Create(get.Path)
if err != nil {
if get.Path == "" {
w = os.Stdout
} else {
return
}
}
defer w.Close()
r, header, err := b.GetReader(get.Key, conf)
if err != nil {
return
}
if _, err = io.Copy(w, r); err != nil {
return
}
if err = r.Close(); err != nil {
return
}
log.Println("Headers: ", header)
if get.Debug {
debug()
}
return
}
func (get *getOpts) Execute(args []string) (err error) {
conf := new(s3gof3r.Config)
*conf = *s3gof3r.DefaultConfig
k, err := getAWSKeys()
if err != nil {
return
}
s3 := s3gof3r.New(get.EndPoint, k)
b := s3.Bucket(get.Bucket)
conf.Concurrency = get.Concurrency
if get.NoSSL {
conf.Scheme = "http"
}
conf.PartSize = get.PartSize
conf.Md5Check = !get.NoMd5
s3gof3r.SetLogger(os.Stderr, "", log.LstdFlags, get.Debug)
if get.VersionID != "" {
get.Key = fmt.Sprintf("%s?versionId=%s", get.Key, get.VersionID)
}
w, err := os.Create(get.Path)
if err != nil {
if get.Path == "" {
w = os.Stdout
} else {
return
}
}
defer w.Close()
r, header, err := b.GetReader(get.Key, conf)
if err != nil {
return
}
if _, err = io.Copy(w, r); err != nil {
return
}
if err = r.Close(); err != nil {
return
}
if get.Debug {
log.Println("Headers: ", header)
}
return
}
func NewS3Remote(config config.Config) (*S3Remote, error) {
s3, err := newS3Client(config)
if err != nil {
return &S3Remote{}, err
}
s3gof3rKeys, err := getS3gof3rKeys(config)
if err != nil {
return &S3Remote{}, err
}
return &S3Remote{
config: config,
BucketName: config.AWS.S3URL.Host,
client: s3,
uploadDownloadClient: s3gof3r.New("", s3gof3rKeys),
}, nil
}
func newUploadDownlaodClient(config config.Config) (*s3gof3r.S3, error) {
s3gKeys, err := getS3gof3rKeys(config)
if err != nil {
return nil, err
}
var s3domain string
// We have to do this due to a recent region related change in s3gof3r:
// https://github.com/rlmcpherson/s3gof3r/blob/b574ee38528c51c2c8652b79e71245817c59bd61/s3gof3r.go#L28-L43
if config.AWS.Region == "us-east-1" {
s3domain = ""
} else {
s3domain = fmt.Sprintf("s3-%v.amazonaws.com", config.AWS.Region)
}
return s3gof3r.New(s3domain, s3gKeys), nil
}
func newS3Output(conf map[string]string) (output, error) {
bucketName := conf["bucket"]
if bucketName == "" {
return nil, errors.New("No bucket specified")
}
fileName := conf["filename"]
if fileName == "" {
return nil, errors.New("No file name specified")
}
keys, err := s3gof3r.EnvKeys()
if err != nil {
return nil, err
}
s3 := s3gof3r.New(conf["endpoint"], keys)
bucket := s3.Bucket(bucketName)
return bucket.PutWriter(fileName, nil, nil)
}
func (put *Put) Execute(args []string) (err error) {
conf := new(s3gof3r.Config)
*conf = *s3gof3r.DefaultConfig
k, err := getAWSKeys()
if err != nil {
return
}
s3 := s3gof3r.New(put.EndPoint, k)
b := s3.Bucket(put.Bucket)
if put.Concurrency > 0 {
conf.Concurrency = put.Concurrency
}
conf.PartSize = put.PartSize
conf.Md5Check = !put.CheckDisable
log.Println(put)
if put.Header == nil {
put.Header = make(http.Header)
}
r, err := os.Open(put.Path)
if err != nil {
if put.Path == "" {
r = os.Stdin
} else {
return
}
}
defer r.Close()
w, err := b.PutWriter(put.Key, put.Header, conf)
if err != nil {
return
}
if _, err = io.Copy(w, r); err != nil {
return
}
if err = w.Close(); err != nil {
return
}
if put.Debug {
debug()
}
return
}
func (put *putOpts) Execute(args []string) (err error) {
conf := new(s3gof3r.Config)
*conf = *s3gof3r.DefaultConfig
k, err := getAWSKeys()
if err != nil {
return
}
s3 := s3gof3r.New(put.EndPoint, k)
b := s3.Bucket(put.Bucket)
conf.Concurrency = put.Concurrency
if put.NoSSL {
conf.Scheme = "http"
}
conf.PartSize = put.PartSize
conf.Md5Check = !put.NoMd5
s3gof3r.SetLogger(os.Stderr, "", log.LstdFlags, put.Debug)
if put.Header == nil {
put.Header = make(http.Header)
}
r, err := os.Open(put.Path)
if err != nil {
if put.Path == "" {
r = os.Stdin
} else {
return
}
}
defer r.Close()
w, err := b.PutWriter(put.Key, put.Header, conf)
if err != nil {
return
}
if _, err = io.Copy(w, r); err != nil {
return
}
if err = w.Close(); err != nil {
return
}
return
}
func NewS3ImageFactory(bucketName string) *ImageFactory {
factory := new(ImageFactory)
//log.Print(imageCollections)
k, err := s3gof3r.EnvKeys() // get S3 keys from environment
if err != nil {
log.Fatal("Unable to init s3", err)
}
// Open bucket to put file into
s3 := s3gof3r.New("", k)
bucket := s3.Bucket(bucketName)
if bucket == nil {
log.Fatal("Unable to init s3", err)
}
factory.NewImage = func(r, o string, b bool) ImageFile {
return NewS3Image(s3, bucket, r, o, b)
}
return factory
}
// GetLegacy ...
func (la *LegacyArguments) GetLegacy() (*Legacy, error) {
// Create a "TEST" snapshot in order to work out which tables are active
// Get a list of Keyspaces and Table Names (plus directories)
// Walk through all the directories.
auth, _ := aws.GetAuth(
la.AwsAccessKey,
la.AwsSecret,
"",
time.Now().AddDate(0, 0, 1))
// Check the bucket exists.
bucket := s3.New(auth, GetAwsRegion(la.AwsRegion)).Bucket(la.S3Bucket)
_, err := bucket.List("/", "/", "", 1)
if err != nil {
return nil, err
}
streamAccess := s3gof3r.New("", s3gof3r.Keys{
AccessKey: la.AwsAccessKey,
SecretKey: la.AwsSecret,
SecurityToken: "",
})
streamBucket := streamAccess.Bucket(la.S3Bucket)
legacy := &Legacy{
DataDirectories: make([]string, 0),
S3Bucket: bucket,
S3StreamBucket: streamBucket,
LogDirectory: la.LogDirectory,
NewSnapshot: la.NewSnapshot,
}
legacy.MachineName, _ = os.Hostname()
legacy.DataDirectories = SplitAndTrim(la.DataDirectories, ",")
legacy.ExcludeKeyspaces = SplitAndTrim(la.ExcludeKeyspaces, ",")
return legacy, nil
}
func UploadToS3(file multipart.File) (id string, err error) {
k, err := s3gof3r.EnvKeys()
if err != nil {
log.Println(err.Error())
return
}
// Open bucket to put file into
s3 := s3gof3r.New(S3Domain, k)
s3.Region()
b := s3.Bucket(S3Bucket)
//Generate unique name
u4, err := uuid.NewV4()
if err != nil {
log.Println(err.Error())
return
}
id = u4.String()
// Open a PutWriter for upload
w, err := b.PutWriter("challenge/"+id, nil, nil)
if err != nil {
log.Println(err.Error())
return
}
if _, err = io.Copy(w, file); err != nil {
log.Println(err.Error())
return
}
if err = w.Close(); err != nil {
log.Println(err.Error())
return
}
return
}
func GetStats(id string) (p Parser, err error) {
k, err := s3gof3r.EnvKeys()
if err != nil {
return
}
s3 := s3gof3r.New(S3Domain, k)
s3.Region()
b := s3.Bucket(S3Bucket)
r, _, err := b.GetReader("challenge/"+id, nil)
if err != nil {
return
}
defer r.Close()
buf := bytes.NewBuffer(nil)
io.Copy(buf, r)
p = WordStats(string(buf.Bytes()))
p.Name = id
return
}
func (cp *cpOpts) Execute(args []string) (err error) {
k, err := getAWSKeys()
if err != nil {
return
}
conf := new(s3gof3r.Config)
*conf = *s3gof3r.DefaultConfig
s3 := s3gof3r.New(cp.EndPoint, k)
conf.Concurrency = cp.Concurrency
if cp.NoSSL {
conf.Scheme = "http"
}
conf.PartSize = cp.PartSize
conf.Md5Check = !cp.NoMd5
s3gof3r.SetLogger(os.Stderr, "", log.LstdFlags, cp.Debug)
// parse positional cp args
if len(args) != 2 {
return fmt.Errorf("cp: source and destination arguments required")
}
var urls [2]*url.URL
for i, a := range args {
urls[i], err = url.Parse(a)
if err != nil {
return fmt.Errorf("parse error: %s", err)
}
if urls[i].Host != "" && urls[i].Scheme != "s3" {
return fmt.Errorf("parse error: %s", urls[i].String())
}
}
src, err := func(src *url.URL) (io.ReadCloser, error) {
if src.Host == "" {
return os.Open(src.Path)
}
r, _, err := s3.Bucket(src.Host).GetReader(src.Path, conf)
return r, err
}(urls[0])
if err != nil {
return
}
dst, err := func(dst *url.URL) (io.WriteCloser, error) {
if dst.Host == "" {
return os.Create(dst.Path)
}
return s3.Bucket(dst.Host).PutWriter(dst.Path, cp.Header, conf)
}(urls[1])
if err != nil {
return
}
if _, err = io.Copy(dst, src); err != nil {
return
}
if err = src.Close(); err != nil {
return
}
if err = dst.Close(); err != nil {
return
}
return
}