func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
var (
origin = config.RequiredString("origin")
cache = config.RequiredString("cache")
kvConf = config.RequiredObject("meta")
maxCacheBytes = config.OptionalInt64("maxCacheBytes", 512<<20)
)
if err := config.Validate(); err != nil {
return nil, err
}
cacheSto, err := ld.GetStorage(cache)
if err != nil {
return nil, err
}
originSto, err := ld.GetStorage(origin)
if err != nil {
return nil, err
}
kv, err := sorted.NewKeyValue(kvConf)
if err != nil {
return nil, err
}
// TODO: enumerate through kv and calculate current size.
// Maybe also even enumerate through cache to see if they match.
// Or even: keep it only in memory and not in kv?
s := &sto{
origin: originSto,
cache: cacheSto,
maxCacheBytes: maxCacheBytes,
kv: kv,
}
return s, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
hostname := config.OptionalString("hostname", "s3.amazonaws.com")
cacheSize := config.OptionalInt64("cacheSize", 32<<20)
client := &s3.Client{
Auth: &s3.Auth{
AccessKey: config.RequiredString("aws_access_key"),
SecretAccessKey: config.RequiredString("aws_secret_access_key"),
Hostname: hostname,
},
}
sto := &s3Storage{
s3Client: client,
bucket: config.RequiredString("bucket"),
hostname: hostname,
}
skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
if err := config.Validate(); err != nil {
return nil, err
}
if cacheSize != 0 {
sto.cache = memory.NewCache(cacheSize)
}
if !skipStartupCheck {
_, err := client.ListBucket(sto.bucket, "", 1)
if serr, ok := err.(*s3.Error); ok {
if serr.AmazonCode == "NoSuchBucket" {
return nil, fmt.Errorf("Bucket %q doesn't exist.", sto.bucket)
}
// This code appears when the hostname has dots in it:
if serr.AmazonCode == "PermanentRedirect" {
loc, lerr := client.BucketLocation(sto.bucket)
if lerr != nil {
return nil, fmt.Errorf("Wrong server for bucket %q; and error determining bucket's location: %v", sto.bucket, lerr)
}
client.Auth.Hostname = loc
_, err = client.ListBucket(sto.bucket, "", 1)
if err == nil {
log.Printf("Warning: s3 server should be %q, not %q. Change config file to avoid start-up latency.", client.Auth.Hostname, hostname)
}
}
// This path occurs when the user set the
// wrong server, or didn't set one at all, but
// the bucket doesn't have dots in it:
if serr.UseEndpoint != "" {
// UseEndpoint will be e.g. "brads3test-ca.s3-us-west-1.amazonaws.com"
// But we only want the "s3-us-west-1.amazonaws.com" part.
client.Auth.Hostname = strings.TrimPrefix(serr.UseEndpoint, sto.bucket+".")
_, err = client.ListBucket(sto.bucket, "", 1)
if err == nil {
log.Printf("Warning: s3 server should be %q, not %q. Change config file to avoid start-up latency.", client.Auth.Hostname, hostname)
}
}
}
if err != nil {
return nil, fmt.Errorf("Error listing bucket %s: %v", sto.bucket, err)
}
}
return sto, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
var (
auth = config.RequiredObject("auth")
bucket = config.RequiredString("bucket")
cacheSize = config.OptionalInt64("cacheSize", 32<<20)
clientID = auth.RequiredString("client_id") // or "auto" for service accounts
clientSecret = auth.OptionalString("client_secret", "")
refreshToken = auth.OptionalString("refresh_token", "")
)
if err := config.Validate(); err != nil {
return nil, err
}
if err := auth.Validate(); err != nil {
return nil, err
}
var dirPrefix string
if parts := strings.SplitN(bucket, "/", 2); len(parts) > 1 {
dirPrefix = parts[1]
bucket = parts[0]
}
if dirPrefix != "" && !strings.HasSuffix(dirPrefix, "/") {
dirPrefix += "/"
}
gs := &Storage{
bucket: bucket,
dirPrefix: dirPrefix,
}
if clientID == "auto" {
var err error
gs.client, err = googlestorage.NewServiceClient()
if err != nil {
return nil, err
}
} else {
if clientSecret == "" {
return nil, errors.New("missing required parameter 'client_secret'")
}
if refreshToken == "" {
return nil, errors.New("missing required parameter 'refresh_token'")
}
oAuthClient := oauth2.NewClient(oauth2.NoContext, oauthutil.NewRefreshTokenSource(&oauth2.Config{
Scopes: []string{googlestorage.Scope},
Endpoint: google.Endpoint,
ClientID: clientID,
ClientSecret: clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}, refreshToken))
gs.client = googlestorage.NewClient(oAuthClient)
}
if cacheSize != 0 {
gs.cache = memory.NewCache(cacheSize)
}
bi, err := gs.client.BucketInfo(bucket)
if err != nil {
return nil, fmt.Errorf("error statting bucket %q: %v", bucket, err)
}
hash := sha1.New()
fmt.Fprintf(hash, "%v%v", bi.TimeCreated, bi.Metageneration)
gs.genRandom = fmt.Sprintf("%x", hash.Sum(nil))
gs.genTime, _ = time.Parse(time.RFC3339, bi.TimeCreated)
return gs, nil
}