func getService(debug bool) *route53.Route53 {
config := aws.Config{}
// ensures throttled requests are retried
config.MaxRetries = aws.Int(100)
if debug {
config.LogLevel = aws.LogLevel(aws.LogDebug)
}
return route53.New(&config)
}
func getService(debug bool, profile string) *route53.Route53 {
config := aws.Config{}
if profile != "" {
config.Credentials = credentials.NewSharedCredentials("", profile)
}
// ensures throttled requests are retried
config.MaxRetries = aws.Int(100)
if debug {
config.LogLevel = aws.LogLevel(aws.LogDebug)
}
return route53.New(session.New(), &config)
}
func NewGoofys(bucket string, awsConfig *aws.Config, flags *FlagStorage) *Goofys {
// Set up the basic struct.
fs := &Goofys{
bucket: bucket,
flags: flags,
umask: 0122,
}
if flags.DebugS3 {
awsConfig.LogLevel = aws.LogLevel(aws.LogDebug | aws.LogDebugWithRequestErrors)
s3Log.Level = logrus.DebugLevel
}
fs.awsConfig = awsConfig
fs.sess = session.New(awsConfig)
fs.s3 = fs.newS3()
err := fs.detectBucketLocation()
if err != nil {
return nil
}
now := time.Now()
fs.rootAttrs = fuseops.InodeAttributes{
Size: 4096,
Nlink: 2,
Mode: flags.DirMode | os.ModeDir,
Atime: now,
Mtime: now,
Ctime: now,
Crtime: now,
Uid: fs.flags.Uid,
Gid: fs.flags.Gid,
}
fs.bufferPool = BufferPool{}.Init()
fs.nextInodeID = fuseops.RootInodeID + 1
fs.inodes = make(map[fuseops.InodeID]*Inode)
root := NewInode(aws.String(""), aws.String(""), flags)
root.Id = fuseops.RootInodeID
root.Attributes = &fs.rootAttrs
fs.inodes[fuseops.RootInodeID] = root
fs.inodesCache = make(map[string]*Inode)
fs.nextHandleID = 1
fs.dirHandles = make(map[fuseops.HandleID]*DirHandle)
fs.fileHandles = make(map[fuseops.HandleID]*FileHandle)
return fs
}
func getConfig(c *cli.Context) *aws.Config {
debug := c.Bool("debug")
profile := c.String("profile")
config := aws.Config{}
if profile != "" {
config.Credentials = credentials.NewSharedCredentials("", profile)
}
// ensures throttled requests are retried
config.MaxRetries = aws.Int(100)
if debug {
config.LogLevel = aws.LogLevel(aws.LogDebug)
}
return &config
}
func NewGoofys(bucket string, awsConfig *aws.Config, flags *flagStorage) *Goofys {
// Set up the basic struct.
fs := &Goofys{
bucket: bucket,
flags: flags,
umask: 0122,
}
if flags.DebugS3 {
awsConfig.LogLevel = aws.LogLevel(aws.LogDebug)
}
fs.awsConfig = awsConfig
fs.s3 = s3.New(awsConfig)
params := &s3.GetBucketLocationInput{Bucket: &bucket}
resp, err := fs.s3.GetBucketLocation(params)
var fromRegion, toRegion string
if err != nil {
if mapAwsError(err) == fuse.ENOENT {
log.Printf("bucket %v does not exist", bucket)
return nil
}
fromRegion, toRegion = parseRegionError(err)
} else {
fs.logS3(resp)
if resp.LocationConstraint == nil {
toRegion = "us-east-1"
} else {
toRegion = *resp.LocationConstraint
}
fromRegion = *awsConfig.Region
}
if len(toRegion) != 0 && fromRegion != toRegion {
log.Printf("Switching from region '%v' to '%v'", fromRegion, toRegion)
awsConfig.Region = &toRegion
fs.s3 = s3.New(awsConfig)
_, err = fs.s3.GetBucketLocation(params)
if err != nil {
log.Println(err)
return nil
}
} else if len(toRegion) == 0 && *awsConfig.Region != "milkyway" {
log.Printf("Unable to detect bucket region, staying at '%v'", *awsConfig.Region)
}
now := time.Now()
fs.rootAttrs = fuseops.InodeAttributes{
Size: 4096,
Nlink: 2,
Mode: flags.DirMode | os.ModeDir,
Atime: now,
Mtime: now,
Ctime: now,
Crtime: now,
Uid: fs.flags.Uid,
Gid: fs.flags.Gid,
}
fs.bufferPool = NewBufferPool(100*1024*1024, 20*1024*1024)
fs.nextInodeID = fuseops.RootInodeID + 1
fs.inodes = make(map[fuseops.InodeID]*Inode)
root := NewInode(aws.String(""), aws.String(""), flags)
root.Id = fuseops.RootInodeID
root.Attributes = &fs.rootAttrs
fs.inodes[fuseops.RootInodeID] = root
fs.inodesCache = make(map[string]*Inode)
fs.nextHandleID = 1
fs.dirHandles = make(map[fuseops.HandleID]*DirHandle)
fs.fileHandles = make(map[fuseops.HandleID]*FileHandle)
return fs
}
func NewGoofys(bucket string, awsConfig *aws.Config, flags *FlagStorage) *Goofys {
// Set up the basic struct.
fs := &Goofys{
bucket: bucket,
flags: flags,
umask: 0122,
}
colon := strings.Index(bucket, ":")
if colon != -1 {
fs.prefix = bucket[colon+1:]
fs.prefix += "/"
for strings.HasSuffix(fs.prefix, "//") {
fs.prefix = fs.prefix[0 : len(fs.prefix)-1]
}
fs.bucket = bucket[0:colon]
bucket = fs.bucket
}
if flags.DebugS3 {
awsConfig.LogLevel = aws.LogLevel(aws.LogDebug | aws.LogDebugWithRequestErrors)
s3Log.Level = logrus.DebugLevel
}
fs.awsConfig = awsConfig
fs.sess = session.New(awsConfig)
fs.s3 = fs.newS3()
if !fs.flags.RegionSet {
err := fs.detectBucketLocationByHEAD()
if err == nil {
// we detected a region header, this is probably AWS S3,
// or we can use anonymous access, or both
fs.sess = session.New(awsConfig)
fs.s3 = fs.newS3()
} else if err == fuse.ENOENT {
log.Errorf("bucket %v does not exist", fs.bucket)
return nil
} else {
// this is NOT AWS, we expect the request to fail with 403 if this is not
// an anonymous bucket, or if the provider doesn't support v4 signing, or both
// swift3 and ceph-s3 return 400 so we know we can fallback to v2 signing
// EMC returns 403 because it doesn't support v4 signing
// minio returns 403 because we are using anonymous credential
if err == fuse.EINVAL {
fs.fallbackV2Signer()
}
}
}
// try again with the credential to make sure
err := mapAwsError(fs.testBucket())
if err != nil {
if err == syscall.EACCES {
// if we still get EACCES, this could be EMC and we should try again
fs.fallbackV2Signer()
err = mapAwsError(fs.testBucket())
}
if err != nil {
log.Errorf("Unable to access '%v': %v", fs.bucket, err)
return nil
}
}
go fs.cleanUpOldMPU()
if flags.UseKMS {
//SSE header string for KMS server-side encryption (SSE-KMS)
fs.sseType = s3.ServerSideEncryptionAwsKms
} else if flags.UseSSE {
//SSE header string for non-KMS server-side encryption (SSE-S3)
fs.sseType = s3.ServerSideEncryptionAes256
}
now := time.Now()
fs.rootAttrs = fuseops.InodeAttributes{
Size: 4096,
Nlink: 2,
Mode: flags.DirMode | os.ModeDir,
Atime: now,
Mtime: now,
Ctime: now,
Crtime: now,
Uid: fs.flags.Uid,
Gid: fs.flags.Gid,
}
fs.bufferPool = BufferPool{}.Init()
fs.nextInodeID = fuseops.RootInodeID + 1
fs.inodes = make(map[fuseops.InodeID]*Inode)
root := NewInode(aws.String(""), aws.String(""), flags)
root.Id = fuseops.RootInodeID
root.Attributes = &fs.rootAttrs
fs.inodes[fuseops.RootInodeID] = root
fs.inodesCache = make(map[string]*Inode)
fs.nextHandleID = 1
fs.dirHandles = make(map[fuseops.HandleID]*DirHandle)
fs.fileHandles = make(map[fuseops.HandleID]*FileHandle)
fs.replicators = Ticket{Total: 16}.Init()
fs.restorers = Ticket{Total: 8}.Init()
return fs
}
