func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
var (
origin = config.RequiredString("origin")
cache = config.RequiredString("cache")
kvConf = config.RequiredObject("meta")
maxCacheBytes = config.OptionalInt64("maxCacheBytes", 512<<20)
)
if err := config.Validate(); err != nil {
return nil, err
}
cacheSto, err := ld.GetStorage(cache)
if err != nil {
return nil, err
}
originSto, err := ld.GetStorage(origin)
if err != nil {
return nil, err
}
kv, err := sorted.NewKeyValue(kvConf)
if err != nil {
return nil, err
}
// TODO: enumerate through kv and calculate current size.
// Maybe also even enumerate through cache to see if they match.
// Or even: keep it only in memory and not in kv?
s := &sto{
origin: originSto,
cache: cacheSto,
maxCacheBytes: maxCacheBytes,
kv: kv,
}
return s, nil
}
// convertToMultiServers takes an old style single-server client configuration and maps it to new a multi-servers configuration that is returned.
func convertToMultiServers(conf jsonconfig.Obj) (jsonconfig.Obj, error) {
server := conf.OptionalString("server", "")
if server == "" {
return nil, errors.New("Could not convert config to multi-servers style: no \"server\" key found.")
}
newConf := jsonconfig.Obj{
"servers": map[string]interface{}{
"server": map[string]interface{}{
"auth": conf.OptionalString("auth", ""),
"default": true,
"server": server,
},
},
"identity": conf.OptionalString("identity", ""),
"identitySecretRing": conf.OptionalString("identitySecretRing", ""),
}
if ignoredFiles := conf.OptionalList("ignoredFiles"); ignoredFiles != nil {
var list []interface{}
for _, v := range ignoredFiles {
list = append(list, v)
}
newConf["ignoredFiles"] = list
}
return newConf, nil
}
// newKeyValueFromJSONConfig returns a KeyValue implementation on top of a
// github.com/syndtr/goleveldb/leveldb file.
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
file := cfg.RequiredString("file")
if err := cfg.Validate(); err != nil {
return nil, err
}
strictness := opt.DefaultStrict
if env.IsDev() {
// Be more strict in dev mode.
strictness = opt.StrictAll
}
opts := &opt.Options{
// The default is 10,
// 8 means 2.126% or 1/47th disk check rate,
// 10 means 0.812% error rate (1/2^(bits/1.44)) or 1/123th disk check rate,
// 12 means 0.31% or 1/322th disk check rate.
// TODO(tgulacsi): decide which number is the best here. Till that go with the default.
Filter: filter.NewBloomFilter(10),
Strict: strictness,
}
db, err := leveldb.OpenFile(file, opts)
if err != nil {
return nil, err
}
is := &kvis{
db: db,
path: file,
opts: opts,
readOpts: &opt.ReadOptions{Strict: strictness},
// On machine crash we want to reindex anyway, and
// fsyncs may impose great performance penalty.
writeOpts: &opt.WriteOptions{Sync: false},
}
return is, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
path := config.RequiredString("path")
if err := config.Validate(); err != nil {
return nil, err
}
return New(path)
}
// printConfigChangeHelp checks if conf contains obsolete keys,
// and prints additional help in this case.
func printConfigChangeHelp(conf jsonconfig.Obj) {
// rename maps from old key names to the new ones.
// If there is no new one, the value is the empty string.
rename := map[string]string{
"keyId": "identity",
"publicKeyBlobref": "",
"selfPubKeyDir": "",
"secretRing": "identitySecretRing",
}
oldConfig := false
configChangedMsg := fmt.Sprintf("The client configuration file (%s) keys have changed.\n", osutil.UserClientConfigPath())
for _, unknown := range conf.UnknownKeys() {
v, ok := rename[unknown]
if ok {
if v != "" {
configChangedMsg += fmt.Sprintf("%q should be renamed %q.\n", unknown, v)
} else {
configChangedMsg += fmt.Sprintf("%q should be removed.\n", unknown)
}
oldConfig = true
}
}
if oldConfig {
configChangedMsg += "Please see http://camlistore.org/docs/client-config, or use camput init to recreate a default one."
log.Print(configChangedMsg)
}
}
func thumbnailerFromConfig(config jsonconfig.Obj) Thumbnailer {
command := config.OptionalList("command")
if len(command) < 1 {
return DefaultThumbnailer
}
return &configThumbnailer{prog: command[0], args: command[1:]}
}
// detectConfigChange returns an informative error if conf contains obsolete keys.
func detectConfigChange(conf jsonconfig.Obj) error {
oldHTTPSKey, oldHTTPSCert := conf.OptionalString("HTTPSKeyFile", ""), conf.OptionalString("HTTPSCertFile", "")
if oldHTTPSKey != "" || oldHTTPSCert != "" {
return fmt.Errorf("Config keys %q and %q have respectively been renamed to %q and %q, please fix your server config.",
"HTTPSKeyFile", "HTTPSCertFile", "httpsKey", "httpsCert")
}
return nil
}
func newStatusFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
if err := conf.Validate(); err != nil {
return nil, err
}
return &StatusHandler{
prefix: ld.MyPrefix(),
handlerFinder: ld,
}, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
sto := &appengineStorage{
namespace: config.OptionalString("namespace", ""),
}
if err := config.Validate(); err != nil {
return nil, err
}
sto.namespace, err = sanitizeNamespace(sto.namespace)
if err != nil {
return nil, err
}
return sto, nil
}
func newKeyValueFromConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
if !compiled {
return nil, ErrNotCompiled
}
file := cfg.RequiredString("file")
if err := cfg.Validate(); err != nil {
return nil, err
}
fi, err := os.Stat(file)
if os.IsNotExist(err) || (err == nil && fi.Size() == 0) {
if err := initDB(file); err != nil {
return nil, fmt.Errorf("could not initialize sqlite DB at %s: %v", file, err)
}
}
db, err := sql.Open("sqlite3", file)
if err != nil {
return nil, err
}
kv := &keyValue{
file: file,
db: db,
KeyValue: &sqlkv.KeyValue{
DB: db,
Gate: syncutil.NewGate(1),
},
}
version, err := kv.SchemaVersion()
if err != nil {
return nil, fmt.Errorf("error getting schema version (need to init database with 'camtool dbinit %s'?): %v", file, err)
}
if err := kv.ping(); err != nil {
return nil, err
}
if version != requiredSchemaVersion {
if env.IsDev() {
// Good signal that we're using the devcam server, so help out
// the user with a more useful tip:
return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion)
}
return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)",
version, requiredSchemaVersion)
}
return kv, nil
}
func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (storage blobserver.Storage, err error) {
sto := &condStorage{}
receive := conf.OptionalStringOrObject("write")
read := conf.RequiredString("read")
remove := conf.OptionalString("remove", "")
if err := conf.Validate(); err != nil {
return nil, err
}
if receive != nil {
sto.storageForReceive, err = buildStorageForReceive(ld, receive)
if err != nil {
return
}
}
sto.read, err = ld.GetStorage(read)
if err != nil {
return
}
if remove != "" {
sto.remove, err = ld.GetStorage(remove)
if err != nil {
return
}
}
return sto, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
url := config.RequiredString("url")
auth := config.RequiredString("auth")
skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
if err := config.Validate(); err != nil {
return nil, err
}
client := client.New(url)
if err = client.SetupAuthFromString(auth); err != nil {
return nil, err
}
client.SetLogger(log.New(os.Stderr, "remote", log.LstdFlags))
sto := &remoteStorage{
client: client,
}
if !skipStartupCheck {
// Do a quick dummy operation to check that our credentials are
// correct.
// TODO(bradfitz,mpl): skip this operation smartly if it turns out this is annoying/slow for whatever reason.
c := make(chan blob.SizedRef, 1)
err = sto.EnumerateBlobs(context.TODO(), c, "", 1)
if err != nil {
return nil, err
}
}
return sto, nil
}
func newMemoryIndexFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
if err := config.Validate(); err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
return nil, err
}
ix := NewMemoryIndex()
ix.InitBlobSource(sto)
return ix, err
}
func NewKeyValue(cfg jsonconfig.Obj) (KeyValue, error) {
var s KeyValue
var err error
typ := cfg.RequiredString("type")
ctor, ok := ctors[typ]
if typ != "" && !ok {
return nil, fmt.Errorf("Invalid sorted.KeyValue type %q", typ)
}
if ok {
s, err = ctor(cfg)
if err != nil {
return nil, fmt.Errorf("error from %q KeyValue: %v", typ, err)
}
}
return s, cfg.Validate()
}
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
ins := &instance{
server: cfg.OptionalString("host", "localhost"),
database: cfg.RequiredString("database"),
user: cfg.OptionalString("user", ""),
password: cfg.OptionalString("password", ""),
}
if err := cfg.Validate(); err != nil {
return nil, err
}
db, err := ins.getCollection()
if err != nil {
return nil, err
}
return &keyValue{db: db, session: ins.session}, nil
}
// newKeyValueFromJSONConfig returns a KeyValue implementation on top of a
// github.com/cznic/kv file.
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
file := cfg.RequiredString("file")
if err := cfg.Validate(); err != nil {
return nil, err
}
opts := &kv.Options{}
db, err := kvutil.Open(file, opts)
if err != nil {
return nil, err
}
is := &kvis{
db: db,
opts: opts,
path: file,
}
return is, nil
}
// NewHandler returns a Handler that proxies requests to an app. Start() on the
// Handler starts the app.
// The apiHost must end in a slash and is the camlistored API server for the app
// process to hit.
// The appHandlerPrefix is the URL path prefix on apiHost where the app is mounted.
// It must end in a slash, and be at minimum "/".
// The conf object has the following members, related to the vars described in
// doc/app-environment.txt:
// "program", string, required. File name of the app's program executable. Either
// an absolute path, or the name of a file located in CAMLI_APP_BINDIR or in PATH.
// "backendURL", string, optional. Automatic if absent. It sets CAMLI_APP_BACKEND_URL.
// "appConfig", object, optional. Additional configuration that the app can request from Camlistore.
func NewHandler(conf jsonconfig.Obj, apiHost, appHandlerPrefix string) (*Handler, error) {
// TODO: remove the appHandlerPrefix if/when we change where the app config JSON URL is made available.
name := conf.RequiredString("program")
backendURL := conf.OptionalString("backendURL", "")
appConfig := conf.OptionalObject("appConfig")
// TODO(mpl): add an auth token in the extra config of the dev server config,
// that the hello app can use to setup a status handler than only responds
// to requests with that token.
if err := conf.Validate(); err != nil {
return nil, err
}
if apiHost == "" {
return nil, fmt.Errorf("app: could not initialize Handler for %q: Camlistore apiHost is unknown", name)
}
if appHandlerPrefix == "" {
return nil, fmt.Errorf("app: could not initialize Handler for %q: empty appHandlerPrefix", name)
}
if backendURL == "" {
var err error
// If not specified in the conf, we're dynamically picking the port of the CAMLI_APP_BACKEND_URL
// now (instead of letting the app itself do it), because we need to know it in advance in order
// to set the app handler's proxy.
backendURL, err = randPortBackendURL(apiHost, appHandlerPrefix)
if err != nil {
return nil, err
}
}
username, password := auth.RandToken(20), auth.RandToken(20)
camliAuth := username + ":" + password
basicAuth := auth.NewBasicAuth(username, password)
envVars := map[string]string{
"CAMLI_API_HOST": apiHost,
"CAMLI_AUTH": camliAuth,
"CAMLI_APP_BACKEND_URL": backendURL,
}
if appConfig != nil {
envVars["CAMLI_APP_CONFIG_URL"] = apiHost + strings.TrimPrefix(appHandlerPrefix, "/") + "config.json"
}
proxyURL, err := url.Parse(backendURL)
if err != nil {
return nil, fmt.Errorf("could not parse backendURL %q: %v", backendURL, err)
}
return &Handler{
name: name,
envVars: envVars,
auth: basicAuth,
appConfig: appConfig,
proxy: httputil.NewSingleHostReverseProxy(proxyURL),
backendURL: backendURL,
}, nil
}
// Reads google storage config and creates a Client. Exits on error.
func doConfig(t *testing.T) (gsa *Client, bucket string) {
if *gsConfigPath == "" {
t.Skip("Skipping manual test. Set flag --gs_config_path to test Google Storage.")
}
cf, err := osutil.NewJSONConfigParser().ReadFile(*gsConfigPath)
if err != nil {
t.Fatalf("Failed to read config: %v", err)
}
var config jsonconfig.Obj
config = cf.RequiredObject("gsconf")
if err := cf.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
auth := config.RequiredObject("auth")
bucket = config.RequiredString("bucket")
if err := config.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
gsa = NewClient(oauth2.NewClient(oauth2.NoContext, oauthutil.NewRefreshTokenSource(&oauth2.Config{
Scopes: []string{Scope},
Endpoint: google.Endpoint,
ClientID: auth.RequiredString("client_id"),
ClientSecret: auth.RequiredString("client_secret"),
RedirectURL: oauthutil.TitleBarRedirectURL,
}, auth.RequiredString("refresh_token"))))
if err := auth.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
return
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
var (
path = config.RequiredString("path")
maxFileSize = config.OptionalInt("maxFileSize", 0)
indexConf = config.OptionalObject("metaIndex")
)
if err := config.Validate(); err != nil {
return nil, err
}
return newStorage(path, int64(maxFileSize), indexConf)
}
func newHandlerFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
indexPrefix := conf.RequiredString("index") // TODO: add optional help tips here?
ownerBlobStr := conf.RequiredString("owner")
devBlockStartupPrefix := conf.OptionalString("devBlockStartupOn", "")
slurpToMemory := conf.OptionalBool("slurpToMemory", false)
if err := conf.Validate(); err != nil {
return nil, err
}
if devBlockStartupPrefix != "" {
_, err := ld.GetHandler(devBlockStartupPrefix)
if err != nil {
return nil, fmt.Errorf("search handler references bogus devBlockStartupOn handler %s: %v", devBlockStartupPrefix, err)
}
}
indexHandler, err := ld.GetHandler(indexPrefix)
if err != nil {
return nil, fmt.Errorf("search config references unknown handler %q", indexPrefix)
}
indexer, ok := indexHandler.(index.Interface)
if !ok {
return nil, fmt.Errorf("search config references invalid indexer %q (actually a %T)", indexPrefix, indexHandler)
}
ownerBlobRef, ok := blob.Parse(ownerBlobStr)
if !ok {
return nil, fmt.Errorf("search 'owner' has malformed blobref %q; expecting e.g. sha1-xxxxxxxxxxxx",
ownerBlobStr)
}
h := NewHandler(indexer, ownerBlobRef)
if slurpToMemory {
ii := indexer.(*index.Index)
ii.Lock()
corpus, err := ii.KeepInMemory()
if err != nil {
ii.Unlock()
return nil, fmt.Errorf("error slurping index to memory: %v", err)
}
h.SetCorpus(corpus)
ii.Unlock()
}
return h, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
sto := &shardStorage{
shardPrefixes: config.RequiredList("backends"),
}
if err := config.Validate(); err != nil {
return nil, err
}
if len(sto.shardPrefixes) == 0 {
return nil, errors.New("shard: need at least one shard")
}
sto.shards = make([]blobserver.Storage, len(sto.shardPrefixes))
for i, prefix := range sto.shardPrefixes {
shardSto, err := ld.GetStorage(prefix)
if err != nil {
return nil, err
}
sto.shards[i] = shardSto
}
return sto, nil
}
func indexFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
is := &indexStorage{}
var (
blobPrefix = config.RequiredString("blobSource")
ns = config.OptionalString("namespace", "")
)
if err := config.Validate(); err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
return nil, err
}
is.ns, err = sanitizeNamespace(ns)
if err != nil {
return nil, err
}
ix, err := index.New(is)
if err != nil {
return nil, err
}
ix.BlobSource = sto
ix.KeyFetcher = ix.BlobSource // TODO(bradfitz): global search? something else?
return ix, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
kvConfig := config.RequiredObject("storage")
if err := config.Validate(); err != nil {
return nil, err
}
kv, err := sorted.NewKeyValue(kvConfig)
if err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
return nil, err
}
ix, err := New(kv)
// TODO(mpl): next time we need to do another fix, make a new error
// type that lets us apply the needed fix depending on its value or
// something. For now just one value/fix.
if err == errMissingWholeRef {
// TODO: maybe we don't want to do that automatically. Brad says
// we have to think about the case on GCE/CoreOS in particular.
if err := ix.fixMissingWholeRef(sto); err != nil {
ix.Close()
return nil, fmt.Errorf("could not fix missing wholeRef entries: %v", err)
}
ix, err = New(kv)
}
if err != nil {
return nil, err
}
ix.InitBlobSource(sto)
return ix, err
}
func newJSONSignFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
var (
// either a short form ("26F5ABDA") or one the longer forms.
keyId = conf.RequiredString("keyId")
pubKeyDestPrefix = conf.OptionalString("publicKeyDest", "")
secretRing = conf.OptionalString("secretRing", "")
)
if err := conf.Validate(); err != nil {
return nil, err
}
h := &Handler{
secretRing: secretRing,
}
var err error
h.entity, err = jsonsign.EntityFromSecring(keyId, h.secretRingPath())
if err != nil {
return nil, err
}
h.pubKey, err = jsonsign.ArmoredPublicKey(h.entity)
ms := &memory.Storage{}
h.pubKeyBlobRef = blob.SHA1FromString(h.pubKey)
if _, err := ms.ReceiveBlob(h.pubKeyBlobRef, strings.NewReader(h.pubKey)); err != nil {
return nil, fmt.Errorf("could not store pub key blob: %v", err)
}
h.pubKeyFetcher = ms
if pubKeyDestPrefix != "" {
sto, err := ld.GetStorage(pubKeyDestPrefix)
if err != nil {
return nil, err
}
h.pubKeyDest = sto
}
h.pubKeyBlobRefServeSuffix = "camli/" + h.pubKeyBlobRef.String()
h.pubKeyHandler = &gethandler.Handler{
Fetcher: ms,
}
h.signer, err = schema.NewSigner(h.pubKeyBlobRef, strings.NewReader(h.pubKey), h.entity)
if err != nil {
return nil, err
}
return h, nil
}
func newFromConfig(ld blobserver.Loader, cfg jsonconfig.Obj) (http.Handler, error) {
hc := HostConfig{
BaseURL: ld.BaseURL(),
Prefix: ld.MyPrefix(),
}
ClientId := make(map[string]string)
ClientSecret := make(map[string]string)
for k, _ := range importers {
var clientId, clientSecret string
if impConf := cfg.OptionalObject(k); impConf != nil {
clientId = impConf.OptionalString("clientID", "")
clientSecret = impConf.OptionalString("clientSecret", "")
// Special case: allow clientSecret to be of form "clientId:clientSecret"
// if the clientId is empty.
if clientId == "" && strings.Contains(clientSecret, ":") {
if f := strings.SplitN(clientSecret, ":", 2); len(f) == 2 {
clientId, clientSecret = f[0], f[1]
}
}
if err := impConf.Validate(); err != nil {
return nil, fmt.Errorf("Invalid static configuration for importer %q: %v", k, err)
}
ClientId[k] = clientId
ClientSecret[k] = clientSecret
}
}
if err := cfg.Validate(); err != nil {
return nil, err
}
hc.ClientId = ClientId
hc.ClientSecret = ClientSecret
host, err := NewHost(hc)
if err != nil {
return nil, err
}
host.didInit.Add(1)
return host, nil
}
func newShareFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
blobRoot := conf.RequiredString("blobRoot")
if blobRoot == "" {
return nil, errors.New("No blobRoot defined for share handler")
}
if err := conf.Validate(); err != nil {
return nil, err
}
bs, err := ld.GetStorage(blobRoot)
if err != nil {
return nil, fmt.Errorf("failed to get share handler's storage at %q: %v", blobRoot, err)
}
fetcher, ok := bs.(blob.Fetcher)
if !ok {
return nil, errors.New("share handler's storage not a Fetcher.")
}
sh := &shareHandler{
fetcher: fetcher,
log: true,
}
return sh, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
var (
auth = config.RequiredObject("auth")
bucket = config.RequiredString("bucket")
cacheSize = config.OptionalInt64("cacheSize", 32<<20)
accountID = auth.RequiredString("account_id")
appKey = auth.RequiredString("application_key")
)
if err := config.Validate(); err != nil {
return nil, err
}
if err := auth.Validate(); err != nil {
return nil, err
}
var dirPrefix string
if parts := strings.SplitN(bucket, "/", 2); len(parts) > 1 {
dirPrefix = parts[1]
bucket = parts[0]
}
if dirPrefix != "" && !strings.HasSuffix(dirPrefix, "/") {
dirPrefix += "/"
}
cl, err := b2.NewClient(accountID, appKey, nil)
if err != nil {
return nil, err
}
b, err := cl.BucketByName(bucket, true)
if err != nil {
return nil, err
}
s := &Storage{
cl: cl, b: b,
dirPrefix: dirPrefix,
}
if cacheSize != 0 {
s.cache = memory.NewCache(cacheSize)
}
return s, nil
}
func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (blobserver.Storage, error) {
var (
smallPrefix = conf.RequiredString("smallBlobs")
largePrefix = conf.RequiredString("largeBlobs")
metaConf = conf.RequiredObject("metaIndex")
)
if err := conf.Validate(); err != nil {
return nil, err
}
small, err := ld.GetStorage(smallPrefix)
if err != nil {
return nil, fmt.Errorf("failed to load smallBlobs at %s: %v", smallPrefix, err)
}
large, err := ld.GetStorage(largePrefix)
if err != nil {
return nil, fmt.Errorf("failed to load largeBlobs at %s: %v", largePrefix, err)
}
largeSubber, ok := large.(subFetcherStorage)
if !ok {
return nil, fmt.Errorf("largeBlobs at %q of type %T doesn't support fetching sub-ranges of blobs",
largePrefix, large)
}
meta, err := sorted.NewKeyValue(metaConf)
if err != nil {
return nil, fmt.Errorf("failed to setup blobpacked metaIndex: %v", err)
}
sto := &storage{
small: small,
large: largeSubber,
meta: meta,
}
sto.init()
recoveryMu.Lock()
defer recoveryMu.Unlock()
if recovery {
log.Print("Starting recovery of blobpacked index")
if err := meta.Close(); err != nil {
return nil, err
}
if err := sto.reindex(context.TODO(), func() (sorted.KeyValue, error) {
return sorted.NewKeyValue(metaConf)
}); err != nil {
return nil, err
}
return sto, nil
}
// Check for a weird state: zip files exist, but no metadata about them
// is recorded. This is probably a corrupt state, and the user likely
// wants to recover.
if !sto.anyMeta() && sto.anyZipPacks() {
log.Fatal("Error: blobpacked storage detects non-zero packed zips, but no metadata. Please re-start in recovery mode with -recovery.")
}
return sto, nil
}
func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (blobserver.Storage, error) {
var (
smallPrefix = conf.RequiredString("smallBlobs")
largePrefix = conf.RequiredString("largeBlobs")
metaConf = conf.RequiredObject("metaIndex")
)
if err := conf.Validate(); err != nil {
return nil, err
}
small, err := ld.GetStorage(smallPrefix)
if err != nil {
return nil, fmt.Errorf("failed to load smallBlobs at %s: %v", smallPrefix, err)
}
large, err := ld.GetStorage(largePrefix)
if err != nil {
return nil, fmt.Errorf("failed to load largeBlobs at %s: %v", largePrefix, err)
}
largeSubber, ok := large.(subFetcherStorage)
if !ok {
return nil, fmt.Errorf("largeBlobs at %q of type %T doesn't support fetching sub-ranges of blobs",
largePrefix, large)
}
meta, err := sorted.NewKeyValue(metaConf)
if err != nil {
return nil, fmt.Errorf("failed to setup blobpacked metaIndex: %v", err)
}
sto := &storage{
small: small,
large: largeSubber,
meta: meta,
}
sto.init()
// Check for a weird state: zip files exist, but no metadata about them
// is recorded. This is probably a corrupt state, and the user likely
// wants to recover.
if !sto.anyMeta() && sto.anyZipPacks() {
log.Printf("Warning: blobpacked storage detects non-zero packed zips, but no metadata. Please re-start in recovery mode.")
// TODO: add a recovery mode.
// Old TODO was:
// fail with a "known corrupt" message and refuse to
// start unless in recovery mode (perhaps a new environment
// var? or flag passed down?) using StreamBlobs starting at
// "l:". Could even do it automatically if total size is
// small or fast enough? But that's confusing if it only
// sometimes finishes recovery. We probably want various
// server start-up modes anyway: "check", "recover", "garbage
// collect", "readonly". So might as well introduce that
// concept now.
// TODO: test start-up recovery mode, once it works.
}
return sto, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
sto := &replicaStorage{
replicaPrefixes: config.RequiredList("backends"),
readPrefixes: config.OptionalList("readBackends"),
}
nReplicas := len(sto.replicaPrefixes)
sto.minWritesForSuccess = config.OptionalInt("minWritesForSuccess", nReplicas)
if err := config.Validate(); err != nil {
return nil, err
}
if nReplicas == 0 {
return nil, errors.New("replica: need at least one replica")
}
if sto.minWritesForSuccess == 0 {
sto.minWritesForSuccess = nReplicas
}
// readPrefixes defaults to the write prefixes.
if len(sto.readPrefixes) == 0 {
sto.readPrefixes = sto.replicaPrefixes
}
for _, prefix := range sto.replicaPrefixes {
s, err := ld.GetStorage(prefix)
if err != nil {
// If it's not a storage interface, it might be an http Handler
// that also supports being a target (e.g. a sync handler).
h, _ := ld.GetHandler(prefix)
var ok bool
if s, ok = h.(blobserver.Storage); !ok {
return nil, err
}
}
sto.replicas = append(sto.replicas, s)
}
for _, prefix := range sto.readPrefixes {
s, err := ld.GetStorage(prefix)
if err != nil {
return nil, err
}
sto.readReplicas = append(sto.readReplicas, s)
}
return sto, nil
}