func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err os.Error) {
client := &s3.Client{
Auth: &s3.Auth{
AccessKey: config.RequiredString("aws_access_key"),
SecretAccessKey: config.RequiredString("aws_secret_access_key"),
},
HttpClient: http.DefaultClient,
}
sto := &s3Storage{
SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
s3Client: client,
bucket: config.RequiredString("bucket"),
}
skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
if err := config.Validate(); err != nil {
return nil, err
}
if !skipStartupCheck {
// TODO: skip this check if a file
// ~/.camli/.configcheck/sha1-("IS GOOD: s3: sha1(access key +
// secret key)") exists and has recent time?
if _, err := client.Buckets(); err != nil {
return nil, fmt.Errorf("Failed to get bucket list from S3: %v", err)
}
}
return sto, nil
}
func newRootFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err os.Error) {
root := &RootHandler{}
root.Stealth = conf.OptionalBool("stealth", false)
if err = conf.Validate(); err != nil {
return
}
return root, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err os.Error) {
sto := &appengineStorage{
SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
}
if err := config.Validate(); err != nil {
return nil, err
}
return sto, nil
}
func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (storage blobserver.Storage, err os.Error) {
sto := &condStorage{
SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
}
receive := conf.OptionalStringOrObject("write")
read := conf.RequiredString("read")
remove := conf.OptionalString("remove", "")
if err := conf.Validate(); err != nil {
return nil, err
}
if receive != nil {
sto.storageForReceive, err = buildStorageForReceive(ld, receive)
if err != nil {
return
}
}
sto.read, err = ld.GetStorage(read)
if err != nil {
return
}
if remove != "" {
sto.remove, err = ld.GetStorage(remove)
if err != nil {
return
}
}
return sto, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err os.Error) {
sto := &DiskStorage{
SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
root: config.RequiredString("path"),
}
if err := config.Validate(); err != nil {
return nil, err
}
fi, err := os.Stat(sto.root)
if err != nil {
return nil, fmt.Errorf("Failed to stat directory %q: %v", sto.root, err)
}
if !fi.IsDirectory() {
return nil, fmt.Errorf("Path %q isn't a directory", sto.root)
}
return sto, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err os.Error) {
sto := &shardStorage{
SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
}
sto.shardPrefixes = config.RequiredList("backends")
if err := config.Validate(); err != nil {
return nil, err
}
if len(sto.shardPrefixes) == 0 {
return nil, os.NewError("shard: need at least one shard")
}
sto.shards = make([]blobserver.Storage, len(sto.shardPrefixes))
for i, prefix := range sto.shardPrefixes {
shardSto, err := ld.GetStorage(prefix)
if err != nil {
return nil, err
}
sto.shards[i] = shardSto
}
return sto, nil
}
func newHandlerFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, os.Error) {
indexPrefix := conf.RequiredString("index") // TODO: add optional help tips here?
ownerBlobStr := conf.RequiredString("owner")
if err := conf.Validate(); err != nil {
return nil, err
}
indexHandler, err := ld.GetHandler(indexPrefix)
if err != nil {
return nil, fmt.Errorf("search config references unknown handler %q", indexPrefix)
}
indexer, ok := indexHandler.(Index)
if !ok {
return nil, fmt.Errorf("search config references invalid indexer %q (actually a %T)", indexPrefix, indexHandler)
}
ownerBlobRef := blobref.Parse(ownerBlobStr)
if ownerBlobRef == nil {
return nil, fmt.Errorf("search 'owner' has malformed blobref %q; expecting e.g. sha1-xxxxxxxxxxxx",
ownerBlobStr)
}
return &Handler{
index: indexer,
owner: ownerBlobRef,
}, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err os.Error) {
sto := &replicaStorage{
SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
}
sto.replicaPrefixes = config.RequiredList("backends")
nReplicas := len(sto.replicaPrefixes)
sto.minWritesForSuccess = config.OptionalInt("minWritesForSuccess", nReplicas)
if err := config.Validate(); err != nil {
return nil, err
}
if nReplicas == 0 {
return nil, os.NewError("replica: need at least one replica")
}
if sto.minWritesForSuccess == 0 {
sto.minWritesForSuccess = nReplicas
}
sto.replicas = make([]blobserver.Storage, nReplicas)
for i, prefix := range sto.replicaPrefixes {
replicaSto, err := ld.GetStorage(prefix)
if err != nil {
return nil, err
}
sto.replicas[i] = replicaSto
}
return sto, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err os.Error) {
url := config.RequiredString("url")
password := config.RequiredString("password")
skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
if err := config.Validate(); err != nil {
return nil, err
}
sto := &remoteStorage{
client: client.New(url, password),
}
if !skipStartupCheck {
// TODO: do a server stat or something to check password
}
return sto, nil
}
func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err os.Error) {
pub := &PublishHandler{}
pub.RootName = conf.RequiredString("rootName")
blobRoot := conf.RequiredString("blobRoot")
searchRoot := conf.RequiredString("searchRoot")
cachePrefix := conf.OptionalString("cache", "")
if err = conf.Validate(); err != nil {
return
}
if pub.RootName == "" {
return nil, os.NewError("invalid empty rootName")
}
bs, err := ld.GetStorage(blobRoot)
if err != nil {
return nil, fmt.Errorf("publish handler's blobRoot of %q error: %v", blobRoot, err)
}
pub.Storage = bs
si, err := ld.GetHandler(searchRoot)
if err != nil {
return nil, fmt.Errorf("publish handler's searchRoot of %q error: %v", searchRoot, err)
}
pub.Search = si.(*search.Handler)
if cachePrefix != "" {
bs, err := ld.GetStorage(cachePrefix)
if err != nil {
return nil, fmt.Errorf("publish handler's cache of %q error: %v", cachePrefix, err)
}
pub.Cache = bs
}
return pub, nil
}
func newJsonSignFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, os.Error) {
pubKeyDestPrefix := conf.OptionalString("publicKeyDest", "")
// either a short form ("26F5ABDA") or one the longer forms.
keyId := conf.RequiredString("keyId")
h := &JSONSignHandler{
secretRing: conf.OptionalString("secretRing", ""),
}
var err os.Error
if err = conf.Validate(); err != nil {
return nil, err
}
h.entity, err = jsonsign.EntityFromSecring(keyId, h.secretRingPath())
if err != nil {
return nil, err
}
armoredPublicKey, err := jsonsign.ArmoredPublicKey(h.entity)
ms := new(blobref.MemoryStore)
h.pubKeyBlobRef, err = ms.AddBlob(crypto.SHA1, armoredPublicKey)
if err != nil {
return nil, err
}
h.pubKeyFetcher = ms
if pubKeyDestPrefix != "" {
sto, err := ld.GetStorage(pubKeyDestPrefix)
if err != nil {
return nil, err
}
h.pubKeyDest = sto
go h.uploadPublicKey(armoredPublicKey)
}
h.pubKeyBlobRefServeSuffix = "camli/" + h.pubKeyBlobRef.String()
h.pubKeyHandler = &handlers.GetHandler{
Fetcher: ms,
AllowGlobalAccess: true, // just public keys
}
return h, nil
}
// Reads google storage config and creates a Client. Exits on error.
func doConfig(t *testing.T) (gsa *Client, bucket string) {
cf, err := jsonconfig.ReadFile("testconfig.json")
if err != nil {
t.Fatalf("Failed to read config: %v", err)
}
var config jsonconfig.Obj
config = cf.RequiredObject("gsconf")
if err := cf.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
auth := config.RequiredObject("auth")
bucket = config.RequiredString("bucket")
if err := config.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
gsa = NewClient(&oauth.Transport{
&oauth.Config{
ClientId: auth.RequiredString("client_id"),
ClientSecret: auth.RequiredString("client_secret"),
Scope: "https://www.googleapis.com/auth/devstorage.read_write",
AuthURL: "https://accounts.google.com/o/oauth2/auth",
TokenURL: "https://accounts.google.com/o/oauth2/token",
RedirectURL: "urn:ietf:wg:oauth:2.0:oob",
},
&oauth.Token{
AccessToken: "",
RefreshToken: auth.RequiredString("refresh_token"),
TokenExpiry: 0,
},
nil,
})
if err := auth.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
return
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, os.Error) {
auth := config.RequiredObject("auth")
gs := &Storage{
&blobserver.SimpleBlobHub{},
config.RequiredString("bucket"),
googlestorage.NewClient(MakeOauthTransport(
auth.RequiredString("client_id"),
auth.RequiredString("client_secret"),
auth.RequiredString("refresh_token"),
)),
}
if err := config.Validate(); err != nil {
return nil, err
}
if err := auth.Validate(); err != nil {
return nil, err
}
return gs, nil
}
func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err os.Error) {
from := conf.RequiredString("from")
to := conf.RequiredString("to")
if err = conf.Validate(); err != nil {
return
}
fromBs, err := ld.GetStorage(from)
if err != nil {
return
}
toBs, err := ld.GetStorage(to)
if err != nil {
return
}
synch, err := createSyncHandler(from, to, fromBs, toBs)
if err != nil {
return
}
return synch, nil
}
func newUiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err os.Error) {
ui := &UIHandler{}
ui.BlobRoot = conf.OptionalString("blobRoot", "")
ui.SearchRoot = conf.OptionalString("searchRoot", "")
ui.JSONSignRoot = conf.OptionalString("jsonSignRoot", "")
pubRoots := conf.OptionalList("publishRoots")
cachePrefix := conf.OptionalString("cache", "")
if err = conf.Validate(); err != nil {
return
}
ui.PublishRoots = make(map[string]*PublishHandler)
for _, pubRoot := range pubRoots {
h, err := ld.GetHandler(pubRoot)
if err != nil {
return nil, fmt.Errorf("UI handler's publishRoots references invalid %q", pubRoot)
}
pubh, ok := h.(*PublishHandler)
if !ok {
return
}
ui.PublishRoots[pubRoot] = pubh
}
checkType := func(key string, htype string) {
v := conf.OptionalString(key, "")
if v == "" {
return
}
ct := ld.GetHandlerType(v)
if ct == "" {
err = fmt.Errorf("UI handler's %q references non-existant %q", key, v)
} else if ct != htype {
err = fmt.Errorf("UI handler's %q references %q of type %q; expected type %q", key, v,
ct, htype)
}
}
checkType("searchRoot", "search")
checkType("jsonSignRoot", "jsonsign")
if err != nil {
return
}
if ui.BlobRoot != "" {
bs, err := ld.GetStorage(ui.BlobRoot)
if err != nil {
return nil, fmt.Errorf("UI handler's blobRoot of %q error: %v", ui.BlobRoot, err)
}
ui.Storage = bs
}
if cachePrefix != "" {
bs, err := ld.GetStorage(cachePrefix)
if err != nil {
return nil, fmt.Errorf("UI handler's cache of %q error: %v", cachePrefix, err)
}
ui.Cache = bs
}
if ui.SearchRoot != "" {
h, _ := ld.GetHandler(ui.SearchRoot)
ui.Search = h.(*search.Handler)
}
ui.staticHandler = http.FileServer(uiFiles)
return ui, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, os.Error) {
blobPrefix := config.RequiredString("blobSource")
indexer := &Indexer{
SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
Host: config.OptionalString("host", "localhost"),
User: config.RequiredString("user"),
Password: config.OptionalString("password", ""),
Database: config.RequiredString("database"),
}
if err := config.Validate(); err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
return nil, err
}
indexer.BlobSource = sto
// Good enough, for now:
indexer.KeyFetcher = indexer.BlobSource
//ownerBlobRef = client.SignerPublicKeyBlobref()
//if ownerBlobRef == nil {
// log.Fatalf("Public key not configured.")
//}
ok, err := indexer.IsAlive()
if !ok {
return nil, fmt.Errorf("Failed to connect to MySQL: %v", err)
}
version, err := indexer.SchemaVersion()
if err != nil {
return nil, fmt.Errorf("error getting schema version (need to init database?): %v", err)
}
if version != requiredSchemaVersion {
if os.Getenv("CAMLI_ADVERTISED_PASSWORD") != "" {
// Good signal that we're using the dev-server script, so help out
// the user with a more useful tip:
return nil, fmt.Errorf("database schema version is %d; expect %d (run \"./dev-server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion)
}
return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)",
version, requiredSchemaVersion)
}
return indexer, nil
}
func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err os.Error) {
ph := &PublishHandler{
bsLoader: ld,
}
ph.RootName = conf.RequiredString("rootName")
ph.JSFiles = conf.OptionalList("js")
ph.CSSFiles = conf.OptionalList("css")
blobRoot := conf.RequiredString("blobRoot")
searchRoot := conf.RequiredString("searchRoot")
cachePrefix := conf.OptionalString("cache", "")
bootstrapSignRoot := conf.OptionalString("devBootstrapPermanodeUsing", "")
if err = conf.Validate(); err != nil {
return
}
if ph.RootName == "" {
return nil, os.NewError("invalid empty rootName")
}
bs, err := ld.GetStorage(blobRoot)
if err != nil {
return nil, fmt.Errorf("publish handler's blobRoot of %q error: %v", blobRoot, err)
}
ph.Storage = bs
si, err := ld.GetHandler(searchRoot)
if err != nil {
return nil, fmt.Errorf("publish handler's searchRoot of %q error: %v", searchRoot, err)
}
var ok bool
ph.Search, ok = si.(*search.Handler)
if !ok {
return nil, fmt.Errorf("publish handler's searchRoot of %q is of type %T, expecting a search handler",
searchRoot, si)
}
if bootstrapSignRoot != "" {
if t := ld.GetHandlerType(bootstrapSignRoot); t != "jsonsign" {
return nil, fmt.Errorf("publish handler's devBootstrapPermanodeUsing must be of type jsonsign")
}
h, _ := ld.GetHandler(bootstrapSignRoot)
jsonSign := h.(*JSONSignHandler)
if err := ph.bootstrapPermanode(jsonSign); err != nil {
return nil, fmt.Errorf("error bootstrapping permanode: %v", err)
}
}
if cachePrefix != "" {
bs, err := ld.GetStorage(cachePrefix)
if err != nil {
return nil, fmt.Errorf("publish handler's cache of %q error: %v", cachePrefix, err)
}
ph.Cache = bs
}
ph.staticHandler = http.FileServer(uiFiles)
return ph, nil
}