func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
var (
origin = config.RequiredString("origin")
cache = config.RequiredString("cache")
kvConf = config.RequiredObject("meta")
maxCacheBytes = config.OptionalInt64("maxCacheBytes", 512<<20)
)
if err := config.Validate(); err != nil {
return nil, err
}
cacheSto, err := ld.GetStorage(cache)
if err != nil {
return nil, err
}
originSto, err := ld.GetStorage(origin)
if err != nil {
return nil, err
}
kv, err := sorted.NewKeyValue(kvConf)
if err != nil {
return nil, err
}
// TODO: enumerate through kv and calculate current size.
// Maybe also even enumerate through cache to see if they match.
// Or even: keep it only in memory and not in kv?
s := &sto{
origin: originSto,
cache: cacheSto,
maxCacheBytes: maxCacheBytes,
kv: kv,
}
return s, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
var (
auth = config.RequiredObject("auth")
bucket = config.RequiredString("bucket")
cacheSize = config.OptionalInt64("cacheSize", 32<<20)
accountID = auth.RequiredString("account_id")
appKey = auth.RequiredString("application_key")
)
if err := config.Validate(); err != nil {
return nil, err
}
if err := auth.Validate(); err != nil {
return nil, err
}
var dirPrefix string
if parts := strings.SplitN(bucket, "/", 2); len(parts) > 1 {
dirPrefix = parts[1]
bucket = parts[0]
}
if dirPrefix != "" && !strings.HasSuffix(dirPrefix, "/") {
dirPrefix += "/"
}
cl, err := b2.NewClient(accountID, appKey, nil)
if err != nil {
return nil, err
}
b, err := cl.BucketByName(bucket, true)
if err != nil {
return nil, err
}
s := &Storage{
cl: cl, b: b,
dirPrefix: dirPrefix,
}
if cacheSize != 0 {
s.cache = memory.NewCache(cacheSize)
}
return s, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
hostname := config.OptionalString("hostname", "s3.amazonaws.com")
cacheSize := config.OptionalInt64("cacheSize", 32<<20)
client := &s3.Client{
Auth: &s3.Auth{
AccessKey: config.RequiredString("aws_access_key"),
SecretAccessKey: config.RequiredString("aws_secret_access_key"),
Hostname: hostname,
},
PutGate: syncutil.NewGate(maxParallelHTTP),
}
bucket := config.RequiredString("bucket")
var dirPrefix string
if parts := strings.SplitN(bucket, "/", 2); len(parts) > 1 {
dirPrefix = parts[1]
bucket = parts[0]
}
if dirPrefix != "" && !strings.HasSuffix(dirPrefix, "/") {
dirPrefix += "/"
}
sto := &s3Storage{
s3Client: client,
bucket: bucket,
dirPrefix: dirPrefix,
hostname: hostname,
}
skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
if err := config.Validate(); err != nil {
return nil, err
}
if cacheSize != 0 {
sto.cache = memory.NewCache(cacheSize)
}
if !skipStartupCheck {
_, err := client.ListBucket(sto.bucket, "", 1)
if serr, ok := err.(*s3.Error); ok {
if serr.AmazonCode == "NoSuchBucket" {
return nil, fmt.Errorf("Bucket %q doesn't exist.", sto.bucket)
}
// This code appears when the hostname has dots in it:
if serr.AmazonCode == "PermanentRedirect" {
loc, lerr := client.BucketLocation(sto.bucket)
if lerr != nil {
return nil, fmt.Errorf("Wrong server for bucket %q; and error determining bucket's location: %v", sto.bucket, lerr)
}
client.Auth.Hostname = loc
_, err = client.ListBucket(sto.bucket, "", 1)
if err == nil {
log.Printf("Warning: s3 server should be %q, not %q. Change config file to avoid start-up latency.", client.Auth.Hostname, hostname)
}
}
// This path occurs when the user set the
// wrong server, or didn't set one at all, but
// the bucket doesn't have dots in it:
if serr.UseEndpoint != "" {
// UseEndpoint will be e.g. "brads3test-ca.s3-us-west-1.amazonaws.com"
// But we only want the "s3-us-west-1.amazonaws.com" part.
client.Auth.Hostname = strings.TrimPrefix(serr.UseEndpoint, sto.bucket+".")
_, err = client.ListBucket(sto.bucket, "", 1)
if err == nil {
log.Printf("Warning: s3 server should be %q, not %q. Change config file to avoid start-up latency.", client.Auth.Hostname, hostname)
}
}
}
if err != nil {
return nil, fmt.Errorf("Error listing bucket %s: %v", sto.bucket, err)
}
}
return sto, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
var (
auth = config.RequiredObject("auth")
bucket = config.RequiredString("bucket")
cacheSize = config.OptionalInt64("cacheSize", 32<<20)
clientID = auth.RequiredString("client_id") // or "auto" for service accounts
clientSecret = auth.OptionalString("client_secret", "")
refreshToken = auth.OptionalString("refresh_token", "")
)
if err := config.Validate(); err != nil {
return nil, err
}
if err := auth.Validate(); err != nil {
return nil, err
}
var dirPrefix string
if parts := strings.SplitN(bucket, "/", 2); len(parts) > 1 {
dirPrefix = parts[1]
bucket = parts[0]
}
if dirPrefix != "" && !strings.HasSuffix(dirPrefix, "/") {
dirPrefix += "/"
}
gs := &Storage{
bucket: bucket,
dirPrefix: dirPrefix,
}
if clientID == "auto" {
var err error
gs.client, err = googlestorage.NewServiceClient()
if err != nil {
return nil, err
}
} else {
if clientSecret == "" {
return nil, errors.New("missing required parameter 'client_secret'")
}
if refreshToken == "" {
return nil, errors.New("missing required parameter 'refresh_token'")
}
oAuthClient := oauth2.NewClient(oauth2.NoContext, oauthutil.NewRefreshTokenSource(&oauth2.Config{
Scopes: []string{googlestorage.Scope},
Endpoint: google.Endpoint,
ClientID: clientID,
ClientSecret: clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}, refreshToken))
gs.client = googlestorage.NewClient(oAuthClient)
}
if cacheSize != 0 {
gs.cache = memory.NewCache(cacheSize)
}
bi, err := gs.client.BucketInfo(bucket)
if err != nil {
return nil, fmt.Errorf("error statting bucket %q: %v", bucket, err)
}
hash := sha1.New()
fmt.Fprintf(hash, "%v%v", bi.TimeCreated, bi.Metageneration)
gs.genRandom = fmt.Sprintf("%x", hash.Sum(nil))
gs.genTime, _ = time.Parse(time.RFC3339, bi.TimeCreated)
return gs, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
var (
auth = config.RequiredObject("auth")
bucket = config.RequiredString("bucket")
cacheSize = config.OptionalInt64("cacheSize", 32<<20)
clientID = auth.RequiredString("client_id") // or "auto" for service accounts
clientSecret = auth.OptionalString("client_secret", "")
refreshToken = auth.OptionalString("refresh_token", "")
)
if err := config.Validate(); err != nil {
return nil, err
}
if err := auth.Validate(); err != nil {
return nil, err
}
var dirPrefix string
if parts := strings.SplitN(bucket, "/", 2); len(parts) > 1 {
dirPrefix = parts[1]
bucket = parts[0]
}
if dirPrefix != "" && !strings.HasSuffix(dirPrefix, "/") {
dirPrefix += "/"
}
gs := &Storage{
bucket: bucket,
dirPrefix: dirPrefix,
}
var (
ctx = context.Background()
ts oauth2.TokenSource
cl *storage.Client
err error
)
if clientID == "auto" {
if !metadata.OnGCE() {
return nil, errors.New(`Cannot use "auto" client_id when not running on GCE`)
}
ts, err = google.DefaultTokenSource(ctx, storage.ScopeReadWrite)
if err != nil {
return nil, err
}
cl, err = storage.NewClient(ctx)
if err != nil {
return nil, err
}
} else {
if clientSecret == "" {
return nil, errors.New("missing required parameter 'client_secret'")
}
if refreshToken == "" {
return nil, errors.New("missing required parameter 'refresh_token'")
}
ts = oauthutil.NewRefreshTokenSource(&oauth2.Config{
Scopes: []string{storage.ScopeReadWrite},
Endpoint: google.Endpoint,
ClientID: clientID,
ClientSecret: clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}, refreshToken)
cl, err = storage.NewClient(ctx, cloud.WithTokenSource(ts))
if err != nil {
return nil, err
}
}
gs.baseHTTPClient = oauth2.NewClient(ctx, ts)
gs.client = cl
if cacheSize != 0 {
gs.cache = memory.NewCache(cacheSize)
}
ba, err := gs.client.Bucket(gs.bucket).Attrs(ctx)
if err != nil {
return nil, fmt.Errorf("error statting bucket %q: %v", gs.bucket, err)
}
hash := sha1.New()
fmt.Fprintf(hash, "%v%v", ba.Created, ba.MetaGeneration)
gs.genRandom = fmt.Sprintf("%x", hash.Sum(nil))
gs.genTime = ba.Created
return gs, nil
}
