Beispiel #1
0
func getService(debug bool) *route53.Route53 {
	config := aws.Config{}
	// ensures throttled requests are retried
	config.MaxRetries = aws.Int(100)
	if debug {
		config.LogLevel = aws.LogLevel(aws.LogDebug)
	}
	return route53.New(&config)
}
Beispiel #2
0
func getService(debug bool, profile string) *route53.Route53 {
	config := aws.Config{}
	if profile != "" {
		config.Credentials = credentials.NewSharedCredentials("", profile)
	}
	// ensures throttled requests are retried
	config.MaxRetries = aws.Int(100)
	if debug {
		config.LogLevel = aws.LogLevel(aws.LogDebug)
	}
	return route53.New(session.New(), &config)
}
Beispiel #3
0
func NewGoofys(bucket string, awsConfig *aws.Config, flags *FlagStorage) *Goofys {
	// Set up the basic struct.
	fs := &Goofys{
		bucket: bucket,
		flags:  flags,
		umask:  0122,
	}

	if flags.DebugS3 {
		awsConfig.LogLevel = aws.LogLevel(aws.LogDebug | aws.LogDebugWithRequestErrors)
		s3Log.Level = logrus.DebugLevel
	}

	fs.awsConfig = awsConfig
	fs.sess = session.New(awsConfig)
	fs.s3 = fs.newS3()

	err := fs.detectBucketLocation()
	if err != nil {
		return nil
	}

	now := time.Now()
	fs.rootAttrs = fuseops.InodeAttributes{
		Size:   4096,
		Nlink:  2,
		Mode:   flags.DirMode | os.ModeDir,
		Atime:  now,
		Mtime:  now,
		Ctime:  now,
		Crtime: now,
		Uid:    fs.flags.Uid,
		Gid:    fs.flags.Gid,
	}

	fs.bufferPool = BufferPool{}.Init()

	fs.nextInodeID = fuseops.RootInodeID + 1
	fs.inodes = make(map[fuseops.InodeID]*Inode)
	root := NewInode(aws.String(""), aws.String(""), flags)
	root.Id = fuseops.RootInodeID
	root.Attributes = &fs.rootAttrs

	fs.inodes[fuseops.RootInodeID] = root
	fs.inodesCache = make(map[string]*Inode)

	fs.nextHandleID = 1
	fs.dirHandles = make(map[fuseops.HandleID]*DirHandle)

	fs.fileHandles = make(map[fuseops.HandleID]*FileHandle)

	return fs
}
Beispiel #4
0
func getConfig(c *cli.Context) *aws.Config {
	debug := c.Bool("debug")
	profile := c.String("profile")
	config := aws.Config{}
	if profile != "" {
		config.Credentials = credentials.NewSharedCredentials("", profile)
	}
	// ensures throttled requests are retried
	config.MaxRetries = aws.Int(100)
	if debug {
		config.LogLevel = aws.LogLevel(aws.LogDebug)
	}
	return &config
}
Beispiel #5
0
func NewGoofys(bucket string, awsConfig *aws.Config, flags *flagStorage) *Goofys {
	// Set up the basic struct.
	fs := &Goofys{
		bucket: bucket,
		flags:  flags,
		umask:  0122,
	}

	if flags.DebugS3 {
		awsConfig.LogLevel = aws.LogLevel(aws.LogDebug)
	}

	fs.awsConfig = awsConfig
	fs.s3 = s3.New(awsConfig)

	params := &s3.GetBucketLocationInput{Bucket: &bucket}
	resp, err := fs.s3.GetBucketLocation(params)
	var fromRegion, toRegion string
	if err != nil {
		if mapAwsError(err) == fuse.ENOENT {
			log.Printf("bucket %v does not exist", bucket)
			return nil
		}
		fromRegion, toRegion = parseRegionError(err)
	} else {
		fs.logS3(resp)

		if resp.LocationConstraint == nil {
			toRegion = "us-east-1"
		} else {
			toRegion = *resp.LocationConstraint
		}

		fromRegion = *awsConfig.Region
	}

	if len(toRegion) != 0 && fromRegion != toRegion {
		log.Printf("Switching from region '%v' to '%v'", fromRegion, toRegion)
		awsConfig.Region = &toRegion
		fs.s3 = s3.New(awsConfig)
		_, err = fs.s3.GetBucketLocation(params)
		if err != nil {
			log.Println(err)
			return nil
		}
	} else if len(toRegion) == 0 && *awsConfig.Region != "milkyway" {
		log.Printf("Unable to detect bucket region, staying at '%v'", *awsConfig.Region)
	}

	now := time.Now()
	fs.rootAttrs = fuseops.InodeAttributes{
		Size:   4096,
		Nlink:  2,
		Mode:   flags.DirMode | os.ModeDir,
		Atime:  now,
		Mtime:  now,
		Ctime:  now,
		Crtime: now,
		Uid:    fs.flags.Uid,
		Gid:    fs.flags.Gid,
	}

	fs.bufferPool = NewBufferPool(100*1024*1024, 20*1024*1024)

	fs.nextInodeID = fuseops.RootInodeID + 1
	fs.inodes = make(map[fuseops.InodeID]*Inode)
	root := NewInode(aws.String(""), aws.String(""), flags)
	root.Id = fuseops.RootInodeID
	root.Attributes = &fs.rootAttrs

	fs.inodes[fuseops.RootInodeID] = root
	fs.inodesCache = make(map[string]*Inode)

	fs.nextHandleID = 1
	fs.dirHandles = make(map[fuseops.HandleID]*DirHandle)

	fs.fileHandles = make(map[fuseops.HandleID]*FileHandle)

	return fs
}
Beispiel #6
0
func NewGoofys(bucket string, awsConfig *aws.Config, flags *FlagStorage) *Goofys {
	// Set up the basic struct.
	fs := &Goofys{
		bucket: bucket,
		flags:  flags,
		umask:  0122,
	}

	colon := strings.Index(bucket, ":")
	if colon != -1 {
		fs.prefix = bucket[colon+1:]
		fs.prefix += "/"
		for strings.HasSuffix(fs.prefix, "//") {
			fs.prefix = fs.prefix[0 : len(fs.prefix)-1]
		}

		fs.bucket = bucket[0:colon]
		bucket = fs.bucket
	}

	if flags.DebugS3 {
		awsConfig.LogLevel = aws.LogLevel(aws.LogDebug | aws.LogDebugWithRequestErrors)
		s3Log.Level = logrus.DebugLevel
	}

	fs.awsConfig = awsConfig
	fs.sess = session.New(awsConfig)
	fs.s3 = fs.newS3()

	if !fs.flags.RegionSet {
		err := fs.detectBucketLocationByHEAD()
		if err == nil {
			// we detected a region header, this is probably AWS S3,
			// or we can use anonymous access, or both
			fs.sess = session.New(awsConfig)
			fs.s3 = fs.newS3()
		} else if err == fuse.ENOENT {
			log.Errorf("bucket %v does not exist", fs.bucket)
			return nil
		} else {
			// this is NOT AWS, we expect the request to fail with 403 if this is not
			// an anonymous bucket, or if the provider doesn't support v4 signing, or both
			// swift3 and ceph-s3 return 400 so we know we can fallback to v2 signing
			// EMC returns 403 because it doesn't support v4 signing
			// minio returns 403 because we are using anonymous credential
			if err == fuse.EINVAL {
				fs.fallbackV2Signer()
			}
		}
	}

	// try again with the credential to make sure
	err := mapAwsError(fs.testBucket())
	if err != nil {
		if err == syscall.EACCES {
			// if we still get EACCES, this could be EMC and we should try again
			fs.fallbackV2Signer()
			err = mapAwsError(fs.testBucket())
		}

		if err != nil {
			log.Errorf("Unable to access '%v': %v", fs.bucket, err)
			return nil
		}
	}

	go fs.cleanUpOldMPU()

	if flags.UseKMS {
		//SSE header string for KMS server-side encryption (SSE-KMS)
		fs.sseType = s3.ServerSideEncryptionAwsKms
	} else if flags.UseSSE {
		//SSE header string for non-KMS server-side encryption (SSE-S3)
		fs.sseType = s3.ServerSideEncryptionAes256
	}

	now := time.Now()
	fs.rootAttrs = fuseops.InodeAttributes{
		Size:   4096,
		Nlink:  2,
		Mode:   flags.DirMode | os.ModeDir,
		Atime:  now,
		Mtime:  now,
		Ctime:  now,
		Crtime: now,
		Uid:    fs.flags.Uid,
		Gid:    fs.flags.Gid,
	}

	fs.bufferPool = BufferPool{}.Init()

	fs.nextInodeID = fuseops.RootInodeID + 1
	fs.inodes = make(map[fuseops.InodeID]*Inode)
	root := NewInode(aws.String(""), aws.String(""), flags)
	root.Id = fuseops.RootInodeID
	root.Attributes = &fs.rootAttrs

	fs.inodes[fuseops.RootInodeID] = root
	fs.inodesCache = make(map[string]*Inode)

	fs.nextHandleID = 1
	fs.dirHandles = make(map[fuseops.HandleID]*DirHandle)

	fs.fileHandles = make(map[fuseops.HandleID]*FileHandle)

	fs.replicators = Ticket{Total: 16}.Init()
	fs.restorers = Ticket{Total: 8}.Init()

	return fs
}