func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
var qs QuadStore
var err error
db, err := bolt.Open(path, 0600, nil)
if err != nil {
glog.Errorln("Error, couldn't open! ", err)
return nil, err
}
qs.db = db
// BoolKey returns false on non-existence. IE, Sync by default.
qs.db.NoSync, _, err = options.BoolKey("nosync")
if err != nil {
return nil, err
}
err = qs.getMetadata()
if err == errNoBucket {
return nil, errors.New("bolt: quadstore has not been initialised")
} else if err != nil {
return nil, err
}
if qs.version != latestDataVersion {
return nil, errors.New("bolt: data version is out of date. Run cayleyupgrade for your config to update the data.")
}
return &qs, nil
}
func createNewMongoGraph(addr string, options graph.Options) error {
conn, err := mgo.Dial(addr)
if err != nil {
return err
}
conn.SetSafe(&mgo.Safe{})
dbName := DefaultDBName
if val, ok := options.StringKey("database_name"); ok {
dbName = val
}
db := conn.DB(dbName)
indexOpts := mgo.Index{
Key: []string{"Sub"},
Unique: false,
DropDups: false,
Background: true,
Sparse: true,
}
db.C("triples").EnsureIndex(indexOpts)
indexOpts.Key = []string{"Pred"}
db.C("triples").EnsureIndex(indexOpts)
indexOpts.Key = []string{"Obj"}
db.C("triples").EnsureIndex(indexOpts)
indexOpts.Key = []string{"Label"}
db.C("triples").EnsureIndex(indexOpts)
return nil
}
func newTripleStore(path string, options graph.Options) (graph.TripleStore, error) {
var ts TripleStore
ts.path = path
cache_size := DefaultCacheSize
if val, ok := options.IntKey("cache_size_mb"); ok {
cache_size = val
}
ts.dbOpts = &opt.Options{
BlockCache: cache.NewLRUCache(cache_size * opt.MiB),
}
ts.dbOpts.ErrorIfMissing = true
write_buffer_mb := DefaultWriteBufferSize
if val, ok := options.IntKey("write_buffer_mb"); ok {
write_buffer_mb = val
}
ts.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
ts.hasher = sha1.New()
ts.writeopts = &opt.WriteOptions{
Sync: false,
}
ts.readopts = &opt.ReadOptions{}
db, err := leveldb.OpenFile(ts.path, ts.dbOpts)
if err != nil {
panic("Error, couldn't open! " + err.Error())
}
ts.db = db
glog.Infoln(ts.GetStats())
ts.getSize()
return &ts, nil
}
func newTripleStore(path string, options graph.Options) (graph.TripleStore, error) {
var qs TripleStore
var err error
qs.path = path
cache_size := DefaultCacheSize
if val, ok := options.IntKey("cache_size_mb"); ok {
cache_size = val
}
qs.dbOpts = &opt.Options{
BlockCache: cache.NewLRUCache(cache_size * opt.MiB),
}
qs.dbOpts.ErrorIfMissing = true
write_buffer_mb := DefaultWriteBufferSize
if val, ok := options.IntKey("write_buffer_mb"); ok {
write_buffer_mb = val
}
qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
qs.writeopts = &opt.WriteOptions{
Sync: false,
}
qs.readopts = &opt.ReadOptions{}
db, err := leveldb.OpenFile(qs.path, qs.dbOpts)
if err != nil {
glog.Errorln("Error, couldn't open! ", err)
return nil, err
}
qs.db = db
glog.Infoln(qs.GetStats())
err = qs.getMetadata()
if err != nil {
return nil, err
}
return &qs, nil
}
func CreateNewMongoGraph(addr string, options graph.Options) bool {
conn, err := mgo.Dial(addr)
if err != nil {
glog.Fatal("Error connecting: ", err)
return false
}
conn.SetSafe(&mgo.Safe{})
dbName := DefaultDBName
if val, ok := options.StringKey("database_name"); ok {
dbName = val
}
db := conn.DB(dbName)
indexOpts := mgo.Index{
Key: []string{"Sub"},
Unique: false,
DropDups: false,
Background: true,
Sparse: true,
}
db.C("triples").EnsureIndex(indexOpts)
indexOpts.Key = []string{"Pred"}
db.C("triples").EnsureIndex(indexOpts)
indexOpts.Key = []string{"Obj"}
db.C("triples").EnsureIndex(indexOpts)
indexOpts.Key = []string{"Provenance"}
db.C("triples").EnsureIndex(indexOpts)
return true
}
func NewSingleReplication(qs graph.QuadStore, opts graph.Options) (graph.QuadWriter, error) {
var (
ignoreMissing bool
ignoreDuplicate bool
err error
)
if *graph.IgnoreMissing {
ignoreMissing = true
} else {
ignoreMissing, _, err = opts.BoolKey("ignore_missing")
if err != nil {
return nil, err
}
}
if *graph.IgnoreDup {
ignoreDuplicate = true
} else {
ignoreDuplicate, _, err = opts.BoolKey("ignore_duplicate")
if err != nil {
return nil, err
}
}
return &Single{
currentID: qs.Horizon(),
qs: qs,
ignoreOpts: graph.IgnoreOpts{
IgnoreDup: ignoreDuplicate,
IgnoreMissing: ignoreMissing,
},
}, nil
}
func newQuadStore(addr string, options graph.Options) (graph.QuadStore, error) {
var qs QuadStore
conn, err := connectSQLTables(addr, options)
if err != nil {
return nil, err
}
localOpt, localOptOk, err := options.BoolKey("local_optimize")
if err != nil {
return nil, err
}
qs.db = conn
qs.sqlFlavor = "postgres"
qs.size = -1
qs.lru = newCache(1024)
// Skip size checking by default.
qs.noSizes = true
if localOptOk {
if localOpt {
qs.noSizes = false
}
}
qs.useEstimates, _, err = options.BoolKey("use_estimates")
if err != nil {
return nil, err
}
return &qs, nil
}
func createSQLTables(addr string, options graph.Options) error {
conn, err := connectSQLTables(addr, options)
if err != nil {
return err
}
defer conn.Close()
tx, err := conn.Begin()
if err != nil {
glog.Errorf("Couldn't begin creation transaction: %s", err)
return err
}
table, err := tx.Exec(nodesTableStatement)
if err != nil {
tx.Rollback()
errd := err.(*pq.Error)
if errd.Code == "42P07" {
return graph.ErrDatabaseExists
}
glog.Errorf("Cannot create nodes table: %v", table)
return err
}
table, err = tx.Exec(`
CREATE TABLE quads (
horizon BIGSERIAL PRIMARY KEY,
subject_hash BYTEA NOT NULL,
predicate_hash BYTEA NOT NULL,
object_hash BYTEA NOT NULL,
label_hash BYTEA,
id BIGINT,
ts timestamp
);`)
if err != nil {
tx.Rollback()
errd := err.(*pq.Error)
if errd.Code == "42P07" {
return graph.ErrDatabaseExists
}
glog.Errorf("Cannot create quad table: %v", table)
return err
}
factor, factorOk, err := options.IntKey("db_fill_factor")
if !factorOk {
factor = defaultFillFactor
}
spoIndexes := quadsSecondaryIndexes(factor)
var index sql.Result
index, err = tx.Exec(quadsUniqueIndex + quadsForeignIndex + spoIndexes)
if err != nil {
glog.Errorf("Cannot create indices: %v", index)
tx.Rollback()
return err
}
tx.Commit()
return nil
}
func createSQLTables(addr string, options graph.Options) error {
conn, err := connectSQLTables(addr, options)
if err != nil {
return err
}
defer conn.Close()
tx, err := conn.Begin()
if err != nil {
glog.Errorf("Couldn't begin creation transaction: %s", err)
return err
}
quadTable, err := tx.Exec(`
CREATE TABLE quads (
subject TEXT NOT NULL,
predicate TEXT NOT NULL,
object TEXT NOT NULL,
label TEXT,
horizon BIGSERIAL PRIMARY KEY,
id BIGINT,
ts timestamp,
subject_hash TEXT NOT NULL,
predicate_hash TEXT NOT NULL,
object_hash TEXT NOT NULL,
label_hash TEXT,
UNIQUE(subject_hash, predicate_hash, object_hash, label_hash)
);`)
if err != nil {
tx.Rollback()
errd := err.(*pq.Error)
if errd.Code == "42P07" {
return graph.ErrDatabaseExists
}
glog.Errorf("Cannot create quad table: %v", quadTable)
return err
}
factor, factorOk, err := options.IntKey("db_fill_factor")
if !factorOk {
factor = 50
}
var index sql.Result
index, err = tx.Exec(fmt.Sprintf(`
CREATE INDEX spo_index ON quads (subject_hash) WITH (FILLFACTOR = %d);
CREATE INDEX pos_index ON quads (predicate_hash) WITH (FILLFACTOR = %d);
CREATE INDEX osp_index ON quads (object_hash) WITH (FILLFACTOR = %d);
`, factor, factor, factor))
if err != nil {
glog.Errorf("Cannot create indices: %v", index)
tx.Rollback()
return err
}
tx.Commit()
return nil
}
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
var qs QuadStore
var err error
qs.path = path
cacheSize := DefaultCacheSize
val, ok, err := options.IntKey("cache_size_mb")
if err != nil {
return nil, err
} else if ok {
cacheSize = val
}
qs.dbOpts = &opt.Options{
BlockCacheCapacity: cacheSize * opt.MiB,
}
qs.dbOpts.ErrorIfMissing = true
writeBufferSize := DefaultWriteBufferSize
val, ok, err = options.IntKey("writeBufferSize")
if err != nil {
return nil, err
} else if ok {
writeBufferSize = val
}
qs.dbOpts.WriteBuffer = writeBufferSize * opt.MiB
qs.writeopts = &opt.WriteOptions{
Sync: false,
}
qs.readopts = &opt.ReadOptions{}
db, err := leveldb.OpenFile(qs.path, qs.dbOpts)
if err != nil {
glog.Errorln("Error, could not open! ", err)
return nil, err
}
qs.db = db
glog.Infoln(qs.GetStats())
vers, err := getVersion(qs.db)
if err != nil {
glog.Errorln("Error, could not read version info! ", err)
db.Close()
return nil, err
} else if vers != latestDataVersion {
return nil, fmt.Errorf("leveldb: data version is out of date (%d vs %d). Run cayleyupgrade for your config to update the data.", vers, latestDataVersion)
}
err = qs.getMetadata()
if err != nil {
db.Close()
return nil, err
}
return &qs, nil
}
func newTripleStore(addr string, options graph.Options) (graph.TripleStore, error) {
var qs TripleStore
conn, err := mgo.Dial(addr)
if err != nil {
return nil, err
}
conn.SetSafe(&mgo.Safe{})
dbName := DefaultDBName
if val, ok := options.StringKey("database_name"); ok {
dbName = val
}
qs.db = conn.DB(dbName)
qs.session = conn
qs.idCache = NewIDLru(1 << 16)
return &qs, nil
}
func createNewMongoGraph(addr string, options graph.Options) error {
conn, err := mgo.Dial(addr)
if err != nil {
return err
}
defer conn.Close()
conn.SetSafe(&mgo.Safe{})
dbName := DefaultDBName
val, ok, err := options.StringKey("database_name")
if err != nil {
return err
} else if ok {
dbName = val
}
db := conn.DB(dbName)
return ensureIndexes(db)
}
func NewTripleStore(addr string, options graph.Options) *TripleStore {
var ts TripleStore
conn, err := mgo.Dial(addr)
if err != nil {
glog.Fatal("Error connecting: ", err)
}
conn.SetSafe(&mgo.Safe{})
dbName := DefaultDBName
if val, ok := options.StringKey("database_name"); ok {
dbName = val
}
ts.db = conn.DB(dbName)
ts.session = conn
ts.hasher = sha1.New()
ts.idCache = NewIDLru(1 << 16)
return &ts
}
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
var qs QuadStore
var err error
db, err := bolt.Open(path, 0600, nil)
if err != nil {
glog.Errorln("Error, couldn't open! ", err)
return nil, err
}
qs.db = db
// BoolKey returns false on non-existence. IE, Sync by default.
qs.db.NoSync, _ = options.BoolKey("nosync")
err = qs.getMetadata()
if err != nil {
return nil, err
}
return &qs, nil
}
func newQuadStore(addr string, options graph.Options) (graph.QuadStore, error) {
var qs QuadStore
conn, err := mgo.Dial(addr)
if err != nil {
return nil, err
}
conn.SetSafe(&mgo.Safe{})
dbName := DefaultDBName
val, ok, err := options.StringKey("database_name")
if err != nil {
return nil, err
} else if ok {
dbName = val
}
qs.db = conn.DB(dbName)
qs.session = conn
qs.ids = newCache(1 << 16)
qs.sizes = newCache(1 << 16)
return &qs, nil
}
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
var qs QuadStore
var err error
qs.path = path
cacheSize := DefaultCacheSize
val, ok, err := options.IntKey("cache_size_mb")
if err != nil {
return nil, err
} else if ok {
cacheSize = val
}
qs.dbOpts = &opt.Options{
BlockCacheCapacity: cacheSize * opt.MiB,
}
qs.dbOpts.ErrorIfMissing = true
writeBufferSize := DefaultWriteBufferSize
val, ok, err = options.IntKey("writeBufferSize")
if err != nil {
return nil, err
} else if ok {
writeBufferSize = val
}
qs.dbOpts.WriteBuffer = writeBufferSize * opt.MiB
qs.writeopts = &opt.WriteOptions{
Sync: false,
}
qs.readopts = &opt.ReadOptions{}
db, err := leveldb.OpenFile(qs.path, qs.dbOpts)
if err != nil {
glog.Errorln("Error, could not open! ", err)
return nil, err
}
qs.db = db
glog.Infoln(qs.GetStats())
err = qs.getMetadata()
if err != nil {
return nil, err
}
return &qs, nil
}
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
var qs QuadStore
var err error
db, err := bolt.Open(path, 0600, nil)
if err != nil {
glog.Errorln("Error, couldn't open! ", err)
return nil, err
}
qs.db = db
// BoolKey returns false on non-existence. IE, Sync by default.
qs.db.NoSync, _, err = options.BoolKey("nosync")
if err != nil {
return nil, err
}
err = qs.getMetadata()
if err == errNoBucket {
panic("bolt: quadstore has not been initialised")
} else if err != nil {
return nil, err
}
return &qs, nil
}
func createNewMongoGraph(addr string, options graph.Options) error {
conn, err := mgo.Dial(addr)
if err != nil {
return err
}
conn.SetSafe(&mgo.Safe{})
dbName := DefaultDBName
val, ok, err := options.StringKey("database_name")
if err != nil {
return err
} else if ok {
dbName = val
}
db := conn.DB(dbName)
indexOpts := mgo.Index{
Key: []string{"subject"},
Unique: false,
DropDups: false,
Background: true,
Sparse: true,
}
db.C("quads").EnsureIndex(indexOpts)
indexOpts.Key = []string{"predicate"}
db.C("quads").EnsureIndex(indexOpts)
indexOpts.Key = []string{"object"}
db.C("quads").EnsureIndex(indexOpts)
indexOpts.Key = []string{"label"}
db.C("quads").EnsureIndex(indexOpts)
logOpts := mgo.Index{
Key: []string{"LogID"},
Unique: true,
DropDups: false,
Background: true,
Sparse: true,
}
db.C("log").EnsureIndex(logOpts)
return nil
}
func createLMDB(path string, opt graph.Options) (env *lmdb.Env, err error) {
err = os.Mkdir(path, 0700)
if err != nil && !os.IsExist(err) {
return nil, err
}
env, err = lmdb.NewEnv()
if err != nil {
return env, err
}
defer func() {
if err != nil {
env.Close()
env = nil
}
}()
maxreaders, _, err := opt.IntKey("maxreaders")
if err != nil {
return nil, err
}
if maxreaders > 0 {
err = env.SetMaxReaders(maxreaders)
if err != nil {
return nil, err
}
}
maxdbs, _, err := opt.IntKey("maxdbs")
if err != nil {
return nil, err
}
if maxdbs == 0 {
maxdbs = 7
}
err = env.SetMaxDBs(maxdbs)
if err != nil {
return nil, err
}
mapsize, _, err := opt.IntKey("mapsize")
if err != nil {
return nil, err
}
err = env.SetMapSize(int64(mapsize))
if err != nil {
return nil, err
}
var flags uint
dbnosync, _, err := opt.BoolKey("nosync")
if err != nil {
return nil, err
}
if dbnosync {
flags |= lmdb.NoSync
}
err = env.Open(path, flags, 0600)
if err != nil {
return nil, err
}
return env, nil
}