// newKeyValueFromJSONConfig returns a KeyValue implementation on top of a
// github.com/syndtr/goleveldb/leveldb file.
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
file := cfg.RequiredString("file")
if err := cfg.Validate(); err != nil {
return nil, err
}
strictness := opt.DefaultStrict
if env.IsDev() {
// Be more strict in dev mode.
strictness = opt.StrictAll
}
opts := &opt.Options{
// The default is 10,
// 8 means 2.126% or 1/47th disk check rate,
// 10 means 0.812% error rate (1/2^(bits/1.44)) or 1/123th disk check rate,
// 12 means 0.31% or 1/322th disk check rate.
// TODO(tgulacsi): decide which number is the best here. Till that go with the default.
Filter: filter.NewBloomFilter(10),
Strict: strictness,
}
db, err := leveldb.OpenFile(file, opts)
if err != nil {
return nil, err
}
is := &kvis{
db: db,
path: file,
opts: opts,
readOpts: &opt.ReadOptions{Strict: strictness},
// On machine crash we want to reindex anyway, and
// fsyncs may impose great performance penalty.
writeOpts: &opt.WriteOptions{Sync: false},
}
return is, nil
}
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
conninfo := fmt.Sprintf("user=%s dbname=%s host=%s password=%s sslmode=%s",
cfg.RequiredString("user"),
cfg.RequiredString("database"),
cfg.OptionalString("host", "localhost"),
cfg.OptionalString("password", ""),
cfg.OptionalString("sslmode", "require"),
)
if err := cfg.Validate(); err != nil {
return nil, err
}
db, err := sql.Open("postgres", conninfo)
if err != nil {
return nil, err
}
for _, tableSql := range SQLCreateTables() {
if _, err := db.Exec(tableSql); err != nil {
return nil, fmt.Errorf("error creating table with %q: %v", tableSql, err)
}
}
for _, statement := range SQLDefineReplace() {
if _, err := db.Exec(statement); err != nil {
return nil, fmt.Errorf("error setting up replace statement with %q: %v", statement, err)
}
}
r, err := db.Query(fmt.Sprintf(`SELECT replaceintometa('version', '%d')`, SchemaVersion()))
if err != nil {
return nil, fmt.Errorf("error setting schema version: %v", err)
}
r.Close()
kv := &keyValue{
db: db,
KeyValue: &sqlkv.KeyValue{
DB: db,
SetFunc: altSet,
BatchSetFunc: altBatchSet,
PlaceHolderFunc: replacePlaceHolders,
},
}
if err := kv.ping(); err != nil {
return nil, fmt.Errorf("PostgreSQL db unreachable: %v", err)
}
version, err := kv.SchemaVersion()
if err != nil {
return nil, fmt.Errorf("error getting schema version (need to init database?): %v", err)
}
if version != requiredSchemaVersion {
if env.IsDev() {
// Good signal that we're using the devcam server, so help out
// the user with a more useful tip:
return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion)
}
return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)",
version, requiredSchemaVersion)
}
return kv, nil
}
func ServeError(rw http.ResponseWriter, req *http.Request, err error) {
rw.WriteHeader(http.StatusInternalServerError)
if IsLocalhost(req) || env.IsDev() {
fmt.Fprintf(rw, "Server error: %s\n", err)
return
}
fmt.Fprintf(rw, "An internal error occured, sorry.")
}
func newKeyValueFromConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
if !compiled {
return nil, ErrNotCompiled
}
file := cfg.RequiredString("file")
if err := cfg.Validate(); err != nil {
return nil, err
}
fi, err := os.Stat(file)
if os.IsNotExist(err) || (err == nil && fi.Size() == 0) {
if err := initDB(file); err != nil {
return nil, fmt.Errorf("could not initialize sqlite DB at %s: %v", file, err)
}
}
db, err := sql.Open("sqlite3", file)
if err != nil {
return nil, err
}
kv := &keyValue{
file: file,
db: db,
KeyValue: &sqlkv.KeyValue{
DB: db,
Gate: syncutil.NewGate(1),
},
}
version, err := kv.SchemaVersion()
if err != nil {
return nil, fmt.Errorf("error getting schema version (need to init database with 'camtool dbinit %s'?): %v", file, err)
}
if err := kv.ping(); err != nil {
return nil, err
}
if version != requiredSchemaVersion {
if env.IsDev() {
// Good signal that we're using the devcam server, so help out
// the user with a more useful tip:
return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion)
}
return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)",
version, requiredSchemaVersion)
}
return kv, nil
}
func init() {
if !env.IsDev() {
// For this particular example importer, we only
// register it if we're in "devcam server" mode.
// Normally you'd avoid this check.
return
}
// This Register call must happen during init.
//
// Register only registers an importer site type and not a
// specific account on a site.
importer.Register("dummy", &imp{})
}
// New returns a new index using the provided key/value storage implementation.
func New(s sorted.KeyValue) (*Index, error) {
idx := &Index{
s: s,
needs: make(map[blob.Ref][]blob.Ref),
neededBy: make(map[blob.Ref][]blob.Ref),
readyReindex: make(map[blob.Ref]bool),
tickleOoo: make(chan bool, 1),
}
if aboutToReindex {
idx.deletes = newDeletionCache()
return idx, nil
}
schemaVersion := idx.schemaVersion()
switch {
case schemaVersion == 0 && idx.isEmpty():
// New index.
err := idx.s.Set(keySchemaVersion.name, fmt.Sprint(requiredSchemaVersion))
if err != nil {
return nil, fmt.Errorf("Could not write index schema version %q: %v", requiredSchemaVersion, err)
}
case schemaVersion != requiredSchemaVersion:
tip := ""
if env.IsDev() {
// Good signal that we're using the devcam server, so help out
// the user with a more useful tip:
tip = `(For the dev server, run "devcam server --wipe" to wipe both your blobs and index)`
} else {
if is4To5SchemaBump(schemaVersion) {
return idx, errMissingWholeRef
}
tip = "Run 'camlistored --reindex' (it might take awhile, but shows status). Alternative: 'camtool dbinit' (or just delete the file for a file based index), and then 'camtool sync --all'"
}
return nil, fmt.Errorf("index schema version is %d; required one is %d. You need to reindex. %s",
schemaVersion, requiredSchemaVersion, tip)
}
if err := idx.initDeletesCache(); err != nil {
return nil, fmt.Errorf("Could not initialize index's deletes cache: %v", err)
}
if err := idx.initNeededMaps(); err != nil {
return nil, fmt.Errorf("Could not initialize index's missing blob maps: %v", err)
}
return idx, nil
}
func (rh *RootHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if wantsDiscovery(req) {
if auth.Allowed(req, auth.OpDiscovery) {
rh.serveDiscovery(rw, req)
return
}
if !rh.Stealth {
http.Error(rw, "Unauthorized", http.StatusUnauthorized)
}
return
}
if rh.Stealth {
return
}
if req.RequestURI == "/" && rh.ui != nil {
http.Redirect(rw, req, "/ui/", http.StatusMovedPermanently)
return
}
if req.URL.Path == "/favicon.ico" {
ServeStaticFile(rw, req, Files, "favicon.ico")
return
}
f := func(p string, a ...interface{}) {
fmt.Fprintf(rw, p, a...)
}
f("<html><body><p>This is camlistored (%s), a "+
"<a href='http://camlistore.org'>Camlistore</a> server.</p>", buildinfo.Version())
if auth.IsLocalhost(req) && !env.IsDev() {
f("<p>If you're coming from localhost, configure your Camlistore server at <a href='/setup'>/setup</a>.</p>")
}
if rh.ui != nil {
f("<p>To manage your content, access the <a href='%s'>%s</a>.</p>", rh.ui.prefix, rh.ui.prefix)
}
if rh.statusRoot != "" {
f("<p>To view status, see <a href='%s'>%s</a>.</p>", rh.statusRoot, rh.statusRoot)
}
if rh.helpRoot != "" {
f("<p>To view more information on accessing the server, see <a href='%s'>%s</a>.</p>", rh.helpRoot, rh.helpRoot)
}
fmt.Fprintf(rw, "</body></html>")
}
// SetupAuth sets the client's authMode. It tries from the environment first if we're on android or in dev mode, and then from the client configuration.
func (c *Client) SetupAuth() error {
if c.paramsOnly {
if c.authMode != nil {
if _, ok := c.authMode.(*auth.None); !ok {
return nil
}
}
return errors.New("client: paramsOnly set; auth should not be configured from config or env vars.")
}
// env var takes precedence, but only if we're in dev mode or on android.
// Too risky otherwise.
if android.OnAndroid() ||
env.IsDev() ||
configDisabled {
authMode, err := auth.FromEnv()
if err == nil {
c.authMode = authMode
return nil
}
if err != auth.ErrNoAuth {
return fmt.Errorf("Could not set up auth from env var CAMLI_AUTH: %v", err)
}
}
if c.server == "" {
return fmt.Errorf("No server defined for this client: can not set up auth.")
}
authConf := serverAuth(c.server)
if authConf == "" {
c.authErr = fmt.Errorf("could not find auth key for server %q in config, defaulting to no auth", c.server)
c.authMode = auth.None{}
return nil
}
var err error
c.authMode, err = auth.FromConfig(authConf)
return err
}
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
var (
user = cfg.RequiredString("user")
database = cfg.RequiredString("database")
host = cfg.OptionalString("host", "")
password = cfg.OptionalString("password", "")
)
if err := cfg.Validate(); err != nil {
return nil, err
}
if !validDatabaseName(database) {
return nil, fmt.Errorf("%q looks like an invalid database name", database)
}
var err error
if host != "" {
host, err = maybeRemapCloudSQL(host)
if err != nil {
return nil, err
}
if !strings.Contains(host, ":") {
host += ":3306"
}
host = "tcp(" + host + ")"
}
// The DSN does NOT have a database name in it so it's
// cacheable and can be shared between different queues & the
// index, all sharing the same database server, cutting down
// number of TCP connections required. We add the database
// name in queries instead.
dsn := fmt.Sprintf("%s:%s@%s/", user, password, host)
db, err := openOrCachedDB(dsn)
if err != nil {
return nil, err
}
if err := CreateDB(db, database); err != nil {
return nil, err
}
if err := createTables(db, database); err != nil {
return nil, err
}
kv := &keyValue{
dsn: dsn,
db: db,
KeyValue: &sqlkv.KeyValue{
DB: db,
TablePrefix: database + ".",
Gate: syncutil.NewGate(20), // arbitrary limit. TODO: configurable, automatically-learned?
},
}
if err := kv.ping(); err != nil {
return nil, fmt.Errorf("MySQL db unreachable: %v", err)
}
version, err := kv.SchemaVersion()
if err != nil {
return nil, fmt.Errorf("error getting current database schema version: %v", err)
}
if version == 0 {
// Newly created table case
if _, err := db.Exec(fmt.Sprintf(`REPLACE INTO %s.meta VALUES ('version', ?)`, database), requiredSchemaVersion); err != nil {
return nil, fmt.Errorf("error setting schema version: %v", err)
}
return kv, nil
}
if version != requiredSchemaVersion {
if version == 20 && requiredSchemaVersion == 21 {
fmt.Fprintf(os.Stderr, fixSchema20to21)
}
if env.IsDev() {
// Good signal that we're using the devcam server, so help out
// the user with a more useful tip:
return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion)
}
return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)",
version, requiredSchemaVersion)
}
return kv, nil
}
func (hl *handlerLoader) initPublisherRootNode(ah *app.Handler) error {
if !env.IsDev() {
return nil
}
h, err := hl.GetHandler("/my-search/")
if err != nil {
return err
}
sh := h.(*search.Handler)
camliRootQuery := func(camliRoot string) (*search.SearchResult, error) {
return sh.Query(&search.SearchQuery{
Limit: 1,
Constraint: &search.Constraint{
Permanode: &search.PermanodeConstraint{
Attr: "camliRoot",
Value: camliRoot,
},
},
})
}
appConfig := ah.AppConfig()
if appConfig == nil {
return errors.New("publisher app handler has no AppConfig")
}
camliRoot, ok := appConfig["camliRoot"].(string)
if !ok {
return fmt.Errorf("camliRoot in publisher app handler appConfig is %T, want string", appConfig["camliRoot"])
}
result, err := camliRootQuery(camliRoot)
if err == nil && len(result.Blobs) > 0 && result.Blobs[0].Blob.Valid() {
// root node found, nothing more to do.
log.Printf("Found %v camliRoot node for publisher: %v", camliRoot, result.Blobs[0].Blob.String())
return nil
}
log.Printf("No %v camliRoot node found, creating one from scratch now.", camliRoot)
bs, err := hl.GetStorage("/bs-recv/")
if err != nil {
return err
}
h, err = hl.GetHandler("/sighelper/")
if err != nil {
return err
}
sigh := h.(*signhandler.Handler)
signUpload := func(bb *schema.Builder) (blob.Ref, error) {
signed, err := sigh.Sign(bb)
if err != nil {
return blob.Ref{}, fmt.Errorf("could not sign blob: %v", err)
}
br := blob.SHA1FromString(signed)
if _, err := blobserver.Receive(bs, br, strings.NewReader(signed)); err != nil {
return blob.Ref{}, fmt.Errorf("could not upload %v: %v", br.String(), err)
}
return br, nil
}
pn, err := signUpload(schema.NewUnsignedPermanode())
if err != nil {
return fmt.Errorf("could not create new camliRoot node: %v", err)
}
if _, err := signUpload(schema.NewSetAttributeClaim(pn, "camliRoot", camliRoot)); err != nil {
return fmt.Errorf("could not set camliRoot on new node %v: %v", pn, err)
}
if _, err := signUpload(schema.NewSetAttributeClaim(pn, "title", "Publish root node for "+camliRoot)); err != nil {
return fmt.Errorf("could not set camliRoot on new node %v: %v", pn, err)
}
return nil
}
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
var (
user = cfg.RequiredString("user")
database = cfg.RequiredString("database")
host = cfg.OptionalString("host", "")
password = cfg.OptionalString("password", "")
)
if err := cfg.Validate(); err != nil {
return nil, err
}
var err error
if host != "" {
host, err = maybeRemapCloudSQL(host)
if err != nil {
return nil, err
}
if !strings.Contains(host, ":") {
host += ":3306"
}
host = "tcp(" + host + ")"
}
// The DSN does NOT have a database name in it so it's
// cacheable and can be shared between different queues & the
// index, all sharing the same database server, cutting down
// number of TCP connections required. We add the database
// name in queries instead.
dsn := fmt.Sprintf("%s:%s@%s/", user, password, host)
db, err := openOrCachedDB(dsn)
if err != nil {
return nil, err
}
if err := CreateDB(db, database); err != nil {
return nil, err
}
for _, tableSQL := range SQLCreateTables() {
tableSQL = strings.Replace(tableSQL, "/*DB*/", database, -1)
if _, err := db.Exec(tableSQL); err != nil {
errMsg := "error creating table with %q: %v."
createError := err
sv, err := serverVersion(db)
if err != nil {
return nil, err
}
if !hasLargeVarchar(sv) {
errMsg += "\nYour MySQL server is too old (< 5.0.3) to support VARCHAR larger than 255."
}
return nil, fmt.Errorf(errMsg, tableSQL, createError)
}
}
if _, err := db.Exec(fmt.Sprintf(`REPLACE INTO %s.meta VALUES ('version', '%d')`, database, SchemaVersion())); err != nil {
return nil, fmt.Errorf("error setting schema version: %v", err)
}
kv := &keyValue{
dsn: dsn,
db: db,
KeyValue: &sqlkv.KeyValue{
DB: db,
TablePrefix: database + ".",
Gate: syncutil.NewGate(20), // arbitrary limit. TODO: configurable, automatically-learned?
},
}
if err := kv.ping(); err != nil {
return nil, fmt.Errorf("MySQL db unreachable: %v", err)
}
version, err := kv.SchemaVersion()
if err != nil {
return nil, fmt.Errorf("error getting schema version (need to init database?): %v", err)
}
if version != requiredSchemaVersion {
if version == 20 && requiredSchemaVersion == 21 {
fmt.Fprintf(os.Stderr, fixSchema20to21)
}
if env.IsDev() {
// Good signal that we're using the devcam server, so help out
// the user with a more useful tip:
return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion)
}
return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)",
version, requiredSchemaVersion)
}
return kv, nil
}