func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) { var qs QuadStore var err error db, err := bolt.Open(path, 0600, nil) if err != nil { glog.Errorln("Error, couldn't open! ", err) return nil, err } qs.db = db // BoolKey returns false on non-existence. IE, Sync by default. qs.db.NoSync, _, err = options.BoolKey("nosync") if err != nil { return nil, err } err = qs.getMetadata() if err == errNoBucket { return nil, errors.New("bolt: quadstore has not been initialised") } else if err != nil { return nil, err } if qs.version != latestDataVersion { return nil, errors.New("bolt: data version is out of date. Run cayleyupgrade for your config to update the data.") } return &qs, nil }
func createNewMongoGraph(addr string, options graph.Options) error { conn, err := mgo.Dial(addr) if err != nil { return err } conn.SetSafe(&mgo.Safe{}) dbName := DefaultDBName if val, ok := options.StringKey("database_name"); ok { dbName = val } db := conn.DB(dbName) indexOpts := mgo.Index{ Key: []string{"Sub"}, Unique: false, DropDups: false, Background: true, Sparse: true, } db.C("triples").EnsureIndex(indexOpts) indexOpts.Key = []string{"Pred"} db.C("triples").EnsureIndex(indexOpts) indexOpts.Key = []string{"Obj"} db.C("triples").EnsureIndex(indexOpts) indexOpts.Key = []string{"Label"} db.C("triples").EnsureIndex(indexOpts) return nil }
func newTripleStore(path string, options graph.Options) (graph.TripleStore, error) { var ts TripleStore ts.path = path cache_size := DefaultCacheSize if val, ok := options.IntKey("cache_size_mb"); ok { cache_size = val } ts.dbOpts = &opt.Options{ BlockCache: cache.NewLRUCache(cache_size * opt.MiB), } ts.dbOpts.ErrorIfMissing = true write_buffer_mb := DefaultWriteBufferSize if val, ok := options.IntKey("write_buffer_mb"); ok { write_buffer_mb = val } ts.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB ts.hasher = sha1.New() ts.writeopts = &opt.WriteOptions{ Sync: false, } ts.readopts = &opt.ReadOptions{} db, err := leveldb.OpenFile(ts.path, ts.dbOpts) if err != nil { panic("Error, couldn't open! " + err.Error()) } ts.db = db glog.Infoln(ts.GetStats()) ts.getSize() return &ts, nil }
func newTripleStore(path string, options graph.Options) (graph.TripleStore, error) { var qs TripleStore var err error qs.path = path cache_size := DefaultCacheSize if val, ok := options.IntKey("cache_size_mb"); ok { cache_size = val } qs.dbOpts = &opt.Options{ BlockCache: cache.NewLRUCache(cache_size * opt.MiB), } qs.dbOpts.ErrorIfMissing = true write_buffer_mb := DefaultWriteBufferSize if val, ok := options.IntKey("write_buffer_mb"); ok { write_buffer_mb = val } qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB qs.writeopts = &opt.WriteOptions{ Sync: false, } qs.readopts = &opt.ReadOptions{} db, err := leveldb.OpenFile(qs.path, qs.dbOpts) if err != nil { glog.Errorln("Error, couldn't open! ", err) return nil, err } qs.db = db glog.Infoln(qs.GetStats()) err = qs.getMetadata() if err != nil { return nil, err } return &qs, nil }
func CreateNewMongoGraph(addr string, options graph.Options) bool { conn, err := mgo.Dial(addr) if err != nil { glog.Fatal("Error connecting: ", err) return false } conn.SetSafe(&mgo.Safe{}) dbName := DefaultDBName if val, ok := options.StringKey("database_name"); ok { dbName = val } db := conn.DB(dbName) indexOpts := mgo.Index{ Key: []string{"Sub"}, Unique: false, DropDups: false, Background: true, Sparse: true, } db.C("triples").EnsureIndex(indexOpts) indexOpts.Key = []string{"Pred"} db.C("triples").EnsureIndex(indexOpts) indexOpts.Key = []string{"Obj"} db.C("triples").EnsureIndex(indexOpts) indexOpts.Key = []string{"Provenance"} db.C("triples").EnsureIndex(indexOpts) return true }
func NewSingleReplication(qs graph.QuadStore, opts graph.Options) (graph.QuadWriter, error) { var ( ignoreMissing bool ignoreDuplicate bool err error ) if *graph.IgnoreMissing { ignoreMissing = true } else { ignoreMissing, _, err = opts.BoolKey("ignore_missing") if err != nil { return nil, err } } if *graph.IgnoreDup { ignoreDuplicate = true } else { ignoreDuplicate, _, err = opts.BoolKey("ignore_duplicate") if err != nil { return nil, err } } return &Single{ currentID: qs.Horizon(), qs: qs, ignoreOpts: graph.IgnoreOpts{ IgnoreDup: ignoreDuplicate, IgnoreMissing: ignoreMissing, }, }, nil }
func newQuadStore(addr string, options graph.Options) (graph.QuadStore, error) { var qs QuadStore conn, err := connectSQLTables(addr, options) if err != nil { return nil, err } localOpt, localOptOk, err := options.BoolKey("local_optimize") if err != nil { return nil, err } qs.db = conn qs.sqlFlavor = "postgres" qs.size = -1 qs.lru = newCache(1024) // Skip size checking by default. qs.noSizes = true if localOptOk { if localOpt { qs.noSizes = false } } qs.useEstimates, _, err = options.BoolKey("use_estimates") if err != nil { return nil, err } return &qs, nil }
func createSQLTables(addr string, options graph.Options) error { conn, err := connectSQLTables(addr, options) if err != nil { return err } defer conn.Close() tx, err := conn.Begin() if err != nil { glog.Errorf("Couldn't begin creation transaction: %s", err) return err } table, err := tx.Exec(nodesTableStatement) if err != nil { tx.Rollback() errd := err.(*pq.Error) if errd.Code == "42P07" { return graph.ErrDatabaseExists } glog.Errorf("Cannot create nodes table: %v", table) return err } table, err = tx.Exec(` CREATE TABLE quads ( horizon BIGSERIAL PRIMARY KEY, subject_hash BYTEA NOT NULL, predicate_hash BYTEA NOT NULL, object_hash BYTEA NOT NULL, label_hash BYTEA, id BIGINT, ts timestamp );`) if err != nil { tx.Rollback() errd := err.(*pq.Error) if errd.Code == "42P07" { return graph.ErrDatabaseExists } glog.Errorf("Cannot create quad table: %v", table) return err } factor, factorOk, err := options.IntKey("db_fill_factor") if !factorOk { factor = defaultFillFactor } spoIndexes := quadsSecondaryIndexes(factor) var index sql.Result index, err = tx.Exec(quadsUniqueIndex + quadsForeignIndex + spoIndexes) if err != nil { glog.Errorf("Cannot create indices: %v", index) tx.Rollback() return err } tx.Commit() return nil }
func createSQLTables(addr string, options graph.Options) error { conn, err := connectSQLTables(addr, options) if err != nil { return err } defer conn.Close() tx, err := conn.Begin() if err != nil { glog.Errorf("Couldn't begin creation transaction: %s", err) return err } quadTable, err := tx.Exec(` CREATE TABLE quads ( subject TEXT NOT NULL, predicate TEXT NOT NULL, object TEXT NOT NULL, label TEXT, horizon BIGSERIAL PRIMARY KEY, id BIGINT, ts timestamp, subject_hash TEXT NOT NULL, predicate_hash TEXT NOT NULL, object_hash TEXT NOT NULL, label_hash TEXT, UNIQUE(subject_hash, predicate_hash, object_hash, label_hash) );`) if err != nil { tx.Rollback() errd := err.(*pq.Error) if errd.Code == "42P07" { return graph.ErrDatabaseExists } glog.Errorf("Cannot create quad table: %v", quadTable) return err } factor, factorOk, err := options.IntKey("db_fill_factor") if !factorOk { factor = 50 } var index sql.Result index, err = tx.Exec(fmt.Sprintf(` CREATE INDEX spo_index ON quads (subject_hash) WITH (FILLFACTOR = %d); CREATE INDEX pos_index ON quads (predicate_hash) WITH (FILLFACTOR = %d); CREATE INDEX osp_index ON quads (object_hash) WITH (FILLFACTOR = %d); `, factor, factor, factor)) if err != nil { glog.Errorf("Cannot create indices: %v", index) tx.Rollback() return err } tx.Commit() return nil }
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) { var qs QuadStore var err error qs.path = path cacheSize := DefaultCacheSize val, ok, err := options.IntKey("cache_size_mb") if err != nil { return nil, err } else if ok { cacheSize = val } qs.dbOpts = &opt.Options{ BlockCacheCapacity: cacheSize * opt.MiB, } qs.dbOpts.ErrorIfMissing = true writeBufferSize := DefaultWriteBufferSize val, ok, err = options.IntKey("writeBufferSize") if err != nil { return nil, err } else if ok { writeBufferSize = val } qs.dbOpts.WriteBuffer = writeBufferSize * opt.MiB qs.writeopts = &opt.WriteOptions{ Sync: false, } qs.readopts = &opt.ReadOptions{} db, err := leveldb.OpenFile(qs.path, qs.dbOpts) if err != nil { glog.Errorln("Error, could not open! ", err) return nil, err } qs.db = db glog.Infoln(qs.GetStats()) vers, err := getVersion(qs.db) if err != nil { glog.Errorln("Error, could not read version info! ", err) db.Close() return nil, err } else if vers != latestDataVersion { return nil, fmt.Errorf("leveldb: data version is out of date (%d vs %d). Run cayleyupgrade for your config to update the data.", vers, latestDataVersion) } err = qs.getMetadata() if err != nil { db.Close() return nil, err } return &qs, nil }
func newTripleStore(addr string, options graph.Options) (graph.TripleStore, error) { var qs TripleStore conn, err := mgo.Dial(addr) if err != nil { return nil, err } conn.SetSafe(&mgo.Safe{}) dbName := DefaultDBName if val, ok := options.StringKey("database_name"); ok { dbName = val } qs.db = conn.DB(dbName) qs.session = conn qs.idCache = NewIDLru(1 << 16) return &qs, nil }
func createNewMongoGraph(addr string, options graph.Options) error { conn, err := mgo.Dial(addr) if err != nil { return err } defer conn.Close() conn.SetSafe(&mgo.Safe{}) dbName := DefaultDBName val, ok, err := options.StringKey("database_name") if err != nil { return err } else if ok { dbName = val } db := conn.DB(dbName) return ensureIndexes(db) }
func NewTripleStore(addr string, options graph.Options) *TripleStore { var ts TripleStore conn, err := mgo.Dial(addr) if err != nil { glog.Fatal("Error connecting: ", err) } conn.SetSafe(&mgo.Safe{}) dbName := DefaultDBName if val, ok := options.StringKey("database_name"); ok { dbName = val } ts.db = conn.DB(dbName) ts.session = conn ts.hasher = sha1.New() ts.idCache = NewIDLru(1 << 16) return &ts }
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) { var qs QuadStore var err error db, err := bolt.Open(path, 0600, nil) if err != nil { glog.Errorln("Error, couldn't open! ", err) return nil, err } qs.db = db // BoolKey returns false on non-existence. IE, Sync by default. qs.db.NoSync, _ = options.BoolKey("nosync") err = qs.getMetadata() if err != nil { return nil, err } return &qs, nil }
func newQuadStore(addr string, options graph.Options) (graph.QuadStore, error) { var qs QuadStore conn, err := mgo.Dial(addr) if err != nil { return nil, err } conn.SetSafe(&mgo.Safe{}) dbName := DefaultDBName val, ok, err := options.StringKey("database_name") if err != nil { return nil, err } else if ok { dbName = val } qs.db = conn.DB(dbName) qs.session = conn qs.ids = newCache(1 << 16) qs.sizes = newCache(1 << 16) return &qs, nil }
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) { var qs QuadStore var err error qs.path = path cacheSize := DefaultCacheSize val, ok, err := options.IntKey("cache_size_mb") if err != nil { return nil, err } else if ok { cacheSize = val } qs.dbOpts = &opt.Options{ BlockCacheCapacity: cacheSize * opt.MiB, } qs.dbOpts.ErrorIfMissing = true writeBufferSize := DefaultWriteBufferSize val, ok, err = options.IntKey("writeBufferSize") if err != nil { return nil, err } else if ok { writeBufferSize = val } qs.dbOpts.WriteBuffer = writeBufferSize * opt.MiB qs.writeopts = &opt.WriteOptions{ Sync: false, } qs.readopts = &opt.ReadOptions{} db, err := leveldb.OpenFile(qs.path, qs.dbOpts) if err != nil { glog.Errorln("Error, could not open! ", err) return nil, err } qs.db = db glog.Infoln(qs.GetStats()) err = qs.getMetadata() if err != nil { return nil, err } return &qs, nil }
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) { var qs QuadStore var err error db, err := bolt.Open(path, 0600, nil) if err != nil { glog.Errorln("Error, couldn't open! ", err) return nil, err } qs.db = db // BoolKey returns false on non-existence. IE, Sync by default. qs.db.NoSync, _, err = options.BoolKey("nosync") if err != nil { return nil, err } err = qs.getMetadata() if err == errNoBucket { panic("bolt: quadstore has not been initialised") } else if err != nil { return nil, err } return &qs, nil }
func createNewMongoGraph(addr string, options graph.Options) error { conn, err := mgo.Dial(addr) if err != nil { return err } conn.SetSafe(&mgo.Safe{}) dbName := DefaultDBName val, ok, err := options.StringKey("database_name") if err != nil { return err } else if ok { dbName = val } db := conn.DB(dbName) indexOpts := mgo.Index{ Key: []string{"subject"}, Unique: false, DropDups: false, Background: true, Sparse: true, } db.C("quads").EnsureIndex(indexOpts) indexOpts.Key = []string{"predicate"} db.C("quads").EnsureIndex(indexOpts) indexOpts.Key = []string{"object"} db.C("quads").EnsureIndex(indexOpts) indexOpts.Key = []string{"label"} db.C("quads").EnsureIndex(indexOpts) logOpts := mgo.Index{ Key: []string{"LogID"}, Unique: true, DropDups: false, Background: true, Sparse: true, } db.C("log").EnsureIndex(logOpts) return nil }
func createLMDB(path string, opt graph.Options) (env *lmdb.Env, err error) { err = os.Mkdir(path, 0700) if err != nil && !os.IsExist(err) { return nil, err } env, err = lmdb.NewEnv() if err != nil { return env, err } defer func() { if err != nil { env.Close() env = nil } }() maxreaders, _, err := opt.IntKey("maxreaders") if err != nil { return nil, err } if maxreaders > 0 { err = env.SetMaxReaders(maxreaders) if err != nil { return nil, err } } maxdbs, _, err := opt.IntKey("maxdbs") if err != nil { return nil, err } if maxdbs == 0 { maxdbs = 7 } err = env.SetMaxDBs(maxdbs) if err != nil { return nil, err } mapsize, _, err := opt.IntKey("mapsize") if err != nil { return nil, err } err = env.SetMapSize(int64(mapsize)) if err != nil { return nil, err } var flags uint dbnosync, _, err := opt.BoolKey("nosync") if err != nil { return nil, err } if dbnosync { flags |= lmdb.NoSync } err = env.Open(path, flags, 0600) if err != nil { return nil, err } return env, nil }