func (qs *QuadStore) WriteHorizonAndSize(tx *bolt.Tx) error {
buf := new(bytes.Buffer)
err := binary.Write(buf, binary.LittleEndian, qs.size)
if err != nil {
glog.Errorf("Couldn't convert size!")
return err
}
b := tx.Bucket(metaBucket)
b.FillPercent = localFillPercent
werr := b.Put([]byte("size"), buf.Bytes())
if werr != nil {
glog.Error("Couldn't write size!")
return werr
}
buf.Reset()
err = binary.Write(buf, binary.LittleEndian, qs.horizon)
if err != nil {
glog.Errorf("Couldn't convert horizon!")
}
werr = b.Put([]byte("horizon"), buf.Bytes())
if werr != nil {
glog.Error("Couldn't write horizon!")
return werr
}
return err
}
func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
_, err := ts.db.Get(ts.createKeyFor("s", "p", "o", t), ts.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Errorf("Couldn't access DB to confirm deletion")
return
}
if err == leveldb.ErrNotFound {
// No such triple in the database, forget about it.
return
}
batch := &leveldb.Batch{}
batch.Delete(ts.createKeyFor("s", "p", "o", t))
batch.Delete(ts.createKeyFor("o", "s", "p", t))
batch.Delete(ts.createKeyFor("p", "o", "s", t))
ts.UpdateValueKeyBy(t.Get("s"), -1, batch)
ts.UpdateValueKeyBy(t.Get("p"), -1, batch)
ts.UpdateValueKeyBy(t.Get("o"), -1, batch)
if t.Get("c") != "" {
batch.Delete(ts.createProvKeyFor("p", "s", "o", t))
ts.UpdateValueKeyBy(t.Get("c"), -1, batch)
}
err = ts.db.Write(batch, nil)
if err != nil {
glog.Errorf("Couldn't delete triple %s", t.ToString())
return
}
ts.size--
}
func (qs *QuadStore) UpdateValueKeyBy(name string, amount int64, tx *bolt.Tx) error {
value := ValueData{name, amount}
b := tx.Bucket(nodeBucket)
b.FillPercent = localFillPercent
key := qs.createValueKeyFor(name)
data := b.Get(key)
if data != nil {
// Node exists in the database -- unmarshal and update.
err := json.Unmarshal(data, &value)
if err != nil {
glog.Errorf("Error: couldn't reconstruct value: %v", err)
return err
}
value.Size += amount
}
// Are we deleting something?
if value.Size <= 0 {
value.Size = 0
}
// Repackage and rewrite.
bytes, err := json.Marshal(&value)
if err != nil {
glog.Errorf("Couldn't write to buffer for value %s: %s", name, err)
return err
}
err = b.Put(key, bytes)
return err
}
func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
_, err := ts.db.Get(ts.createKeyFor(spo, t), ts.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Errorf("Couldn't access DB to confirm deletion")
return
}
if err == leveldb.ErrNotFound {
// No such triple in the database, forget about it.
return
}
batch := &leveldb.Batch{}
batch.Delete(ts.createKeyFor(spo, t))
batch.Delete(ts.createKeyFor(osp, t))
batch.Delete(ts.createKeyFor(pos, t))
ts.UpdateValueKeyBy(t.Get(graph.Subject), -1, batch)
ts.UpdateValueKeyBy(t.Get(graph.Predicate), -1, batch)
ts.UpdateValueKeyBy(t.Get(graph.Object), -1, batch)
if t.Get(graph.Provenance) != "" {
batch.Delete(ts.createProvKeyFor(pso, t))
ts.UpdateValueKeyBy(t.Get(graph.Provenance), -1, batch)
}
err = ts.db.Write(batch, nil)
if err != nil {
glog.Errorf("Couldn't delete triple %s", t)
return
}
ts.size--
}
// WriteHorizonAndSize ??
func (qs *QuadStore) WriteHorizonAndSize(tx *lmdb.Txn) error {
buf := new(bytes.Buffer)
err := binary.Write(buf, binary.LittleEndian, qs.size)
if err != nil {
glog.Errorf("Couldn't convert size!")
return err
}
werr := tx.Put(qs.metaDBI, []byte("size"), buf.Bytes(), 0)
if werr != nil {
glog.Error("Couldn't write size!")
return werr
}
buf.Reset()
err = binary.Write(buf, binary.LittleEndian, qs.horizon)
if err != nil {
glog.Errorf("Couldn't convert horizon!")
}
werr = tx.Put(qs.metaDBI, []byte("horizon"), buf.Bytes(), 0)
if werr != nil {
glog.Error("Couldn't write horizon!")
return werr
}
return err
}
// UpdateValueKeyBy ??
func (qs *QuadStore) UpdateValueKeyBy(name string, amount int64, tx *lmdb.Txn) error {
value := proto.NodeData{
Name: name,
Size_: amount,
}
key := qs.createValueKeyFor(name)
data, err := tx.Get(qs.nodeDBI, key)
if err == nil {
// Node exists in the database -- unmarshal and update.
var oldvalue proto.NodeData
err := oldvalue.Unmarshal(data)
if err != nil {
glog.Errorf("Error: couldn't reconstruct value: %v", err)
return err
}
oldvalue.Size_ += amount
value = oldvalue
}
// Are we deleting something?
if value.Size_ <= 0 {
value.Size_ = 0
}
// Repackage and rewrite.
bytes, err := value.Marshal()
if err != nil {
glog.Errorf("Couldn't write to buffer for value %s: %s", name, err)
return err
}
err = tx.Put(qs.nodeDBI, key, bytes, 0)
return err
}
func (qs *QuadStore) runTxPostgres(tx *sql.Tx, in []graph.Delta, opts graph.IgnoreOpts) error {
allAdds := true
for _, d := range in {
if d.Action != graph.Add {
allAdds = false
}
}
if allAdds {
return qs.copyFrom(tx, in)
}
insert, err := tx.Prepare(`INSERT INTO quads(subject, predicate, object, label, id, ts, subject_hash, predicate_hash, object_hash, label_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`)
defer insert.Close()
if err != nil {
glog.Errorf("Cannot prepare insert statement: %v", err)
return err
}
for _, d := range in {
switch d.Action {
case graph.Add:
_, err := insert.Exec(
d.Quad.Subject,
d.Quad.Predicate,
d.Quad.Object,
d.Quad.Label,
d.ID.Int(),
d.Timestamp,
hashOf(d.Quad.Subject),
hashOf(d.Quad.Predicate),
hashOf(d.Quad.Object),
hashOf(d.Quad.Label),
)
if err != nil {
glog.Errorf("couldn't prepare INSERT statement: %v", err)
return err
}
case graph.Delete:
result, err := tx.Exec(`DELETE FROM quads WHERE subject=$1 and predicate=$2 and object=$3 and label=$4;`,
d.Quad.Subject, d.Quad.Predicate, d.Quad.Object, d.Quad.Label)
if err != nil {
glog.Errorf("couldn't exec DELETE statement: %v", err)
return err
}
affected, err := result.RowsAffected()
if err != nil {
glog.Errorf("couldn't get DELETE RowsAffected: %v", err)
return err
}
if affected != 1 && !opts.IgnoreMissing {
return errors.New("deleting non-existent triple; rolling back")
}
default:
panic("unknown action")
}
}
return nil
}
func createSQLTables(addr string, options graph.Options) error {
conn, err := connectSQLTables(addr, options)
if err != nil {
return err
}
defer conn.Close()
tx, err := conn.Begin()
if err != nil {
glog.Errorf("Couldn't begin creation transaction: %s", err)
return err
}
quadTable, err := tx.Exec(`
CREATE TABLE quads (
subject TEXT NOT NULL,
predicate TEXT NOT NULL,
object TEXT NOT NULL,
label TEXT,
horizon BIGSERIAL PRIMARY KEY,
id BIGINT,
ts timestamp,
subject_hash TEXT NOT NULL,
predicate_hash TEXT NOT NULL,
object_hash TEXT NOT NULL,
label_hash TEXT,
UNIQUE(subject_hash, predicate_hash, object_hash, label_hash)
);`)
if err != nil {
tx.Rollback()
errd := err.(*pq.Error)
if errd.Code == "42P07" {
return graph.ErrDatabaseExists
}
glog.Errorf("Cannot create quad table: %v", quadTable)
return err
}
factor, factorOk, err := options.IntKey("db_fill_factor")
if !factorOk {
factor = 50
}
var index sql.Result
index, err = tx.Exec(fmt.Sprintf(`
CREATE INDEX spo_index ON quads (subject_hash) WITH (FILLFACTOR = %d);
CREATE INDEX pos_index ON quads (predicate_hash) WITH (FILLFACTOR = %d);
CREATE INDEX osp_index ON quads (object_hash) WITH (FILLFACTOR = %d);
`, factor, factor, factor))
if err != nil {
glog.Errorf("Cannot create indices: %v", index)
tx.Rollback()
return err
}
tx.Commit()
return nil
}
func (it *SQLIterator) Contains(v graph.Value) bool {
var err error
if ok, res := it.sql.quickContains(v); ok {
return res
}
err = it.makeCursor(false, v)
if err != nil {
glog.Errorf("Couldn't make query: %v", err)
it.err = err
if it.cursor != nil {
it.cursor.Close()
}
return false
}
it.cols, err = it.cursor.Columns()
if err != nil {
glog.Errorf("Couldn't get columns")
it.err = err
it.cursor.Close()
return false
}
it.resultList = nil
for {
if !it.cursor.Next() {
glog.V(4).Infoln("sql: No next")
err := it.cursor.Err()
if err != nil {
glog.Errorf("Cursor error in SQL: %v", err)
it.err = err
}
it.cursor.Close()
break
}
s, err := scan(it.cursor, len(it.cols))
if err != nil {
it.err = err
it.cursor.Close()
return false
}
it.resultList = append(it.resultList, s)
}
it.cursor.Close()
it.cursor = nil
if len(it.resultList) != 0 {
it.resultIndex = 0
it.buildResult(0)
return true
}
return false
}
func (ts *TripleStore) Close() {
buf := new(bytes.Buffer)
err := binary.Write(buf, binary.LittleEndian, ts.size)
if err == nil {
werr := ts.db.Put([]byte("__size"), buf.Bytes(), ts.writeopts)
if werr != nil {
glog.Errorf("Couldn't write size before closing!")
}
} else {
glog.Errorf("Couldn't convert size before closing!")
}
ts.db.Close()
ts.open = false
}
func connectSQLTables(addr string, _ graph.Options) (*sql.DB, error) {
// TODO(barakmich): Parse options for more friendly addr, other SQLs.
conn, err := sql.Open("postgres", addr)
if err != nil {
glog.Errorf("Couldn't open database at %s: %#v", addr, err)
return nil, err
}
// "Open may just validate its arguments without creating a connection to the database."
// "To verify that the data source name is valid, call Ping."
// Source: http://golang.org/pkg/database/sql/#Open
if err := conn.Ping(); err != nil {
glog.Errorf("Couldn't open database at %s: %#v", addr, err)
return nil, err
}
return conn, nil
}
func createNewLevelDB(path string, _ graph.Options) error {
opts := &opt.Options{}
db, err := leveldb.OpenFile(path, opts)
if err != nil {
glog.Errorf("Error: could not create database: %v", err)
return err
}
defer db.Close()
qs := &QuadStore{}
qs.db = db
qs.writeopts = &opt.WriteOptions{
Sync: true,
}
qs.readopts = &opt.ReadOptions{}
_, err = qs.db.Get([]byte(horizonKey), qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Errorln("couldn't read from leveldb during init")
return err
}
if err != leveldb.ErrNotFound {
return graph.ErrDatabaseExists
}
// Write some metadata
qs.Close()
return nil
}
func (qs *TripleStore) RemoveTriple(t quad.Quad) {
_, err := qs.db.Get(qs.createKeyFor(spo, t), qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Error("Couldn't access DB to confirm deletion")
return
}
if err == leveldb.ErrNotFound {
// No such triple in the database, forget about it.
return
}
batch := &leveldb.Batch{}
batch.Delete(qs.createKeyFor(spo, t))
batch.Delete(qs.createKeyFor(osp, t))
batch.Delete(qs.createKeyFor(pos, t))
qs.UpdateValueKeyBy(t.Get(quad.Subject), -1, batch)
qs.UpdateValueKeyBy(t.Get(quad.Predicate), -1, batch)
qs.UpdateValueKeyBy(t.Get(quad.Object), -1, batch)
if t.Get(quad.Label) != "" {
batch.Delete(qs.createProvKeyFor(pso, t))
qs.UpdateValueKeyBy(t.Get(quad.Label), -1, batch)
}
err = qs.db.Write(batch, nil)
if err != nil {
glog.Errorf("Couldn't delete triple %s.", t)
return
}
qs.size--
}
func createNewBolt(path string, _ graph.Options) error {
db, err := bolt.Open(path, 0600, nil)
if err != nil {
glog.Errorf("Error: couldn't create Bolt database: %v", err)
return err
}
defer db.Close()
qs := &QuadStore{}
qs.db = db
defer qs.Close()
err = qs.getMetadata()
if err != errNoBucket {
return graph.ErrDatabaseExists
}
err = qs.createBuckets()
if err != nil {
return err
}
err = setVersion(qs.db, latestDataVersion)
if err != nil {
return err
}
qs.Close()
return nil
}
func (qs *QuadStore) copyFrom(tx *sql.Tx, in []graph.Delta) error {
stmt, err := tx.Prepare(pq.CopyIn("quads", "subject", "predicate", "object", "label", "id", "ts", "subject_hash", "predicate_hash", "object_hash", "label_hash"))
if err != nil {
return err
}
for _, d := range in {
_, err := stmt.Exec(
d.Quad.Subject,
d.Quad.Predicate,
d.Quad.Object,
d.Quad.Label,
d.ID.Int(),
d.Timestamp,
hashOf(d.Quad.Subject),
hashOf(d.Quad.Predicate),
hashOf(d.Quad.Object),
hashOf(d.Quad.Label),
)
if err != nil {
glog.Errorf("couldn't prepare COPY statement: %v", err)
return err
}
}
_, err = stmt.Exec()
if err != nil {
return err
}
return stmt.Close()
}
func (qs *TripleStore) Size() int64 {
count, err := qs.db.C("triples").Count()
if err != nil {
glog.Errorf("Error: %v", err)
return 0
}
return int64(count)
}
func (qs *QuadStore) Quad(val graph.Value) quad.Quad {
var q quad.Quad
err := qs.db.C("quads").FindId(val.(string)).One(&q)
if err != nil {
glog.Errorf("Error: Couldn't retrieve quad %s %v", val, err)
}
return q
}
func (ts *TripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.Batch) {
value := &ValueData{name, int64(amount)}
key := ts.createValueKeyFor(name)
b, err := ts.db.Get(key, ts.readopts)
// Error getting the node from the database.
if err != nil && err != leveldb.ErrNotFound {
glog.Errorf("Error reading Value %s from the DB\n", name)
return
}
// Node exists in the database -- unmarshal and update.
if b != nil && err != leveldb.ErrNotFound {
err = json.Unmarshal(b, value)
if err != nil {
glog.Errorln("Error: couldn't reconstruct value ", err)
return
}
value.Size += int64(amount)
}
// Are we deleting something?
if amount < 0 {
if value.Size <= 0 {
if batch == nil {
ts.db.Delete(key, ts.writeopts)
} else {
batch.Delete(key)
}
return
}
}
// Repackage and rewrite.
bytes, err := json.Marshal(&value)
if err != nil {
glog.Errorf("Couldn't write to buffer for value %s\n %s", name, err)
return
}
if batch == nil {
ts.db.Put(key, bytes, ts.writeopts)
} else {
batch.Put(key, bytes)
}
}
func (qs *QuadStore) Size() int64 {
// TODO(barakmich): Make size real; store it in the log, and retrieve it.
count, err := qs.db.C("quads").Count()
if err != nil {
glog.Errorf("Error: %v", err)
return 0
}
return int64(count)
}
func (qs *QuadStore) Horizon() graph.PrimaryKey {
var horizon int64
err := qs.db.QueryRow("SELECT horizon FROM quads ORDER BY horizon DESC LIMIT 1;").Scan(&horizon)
if err != nil {
glog.Errorf("Couldn't execute horizon: %v", err)
return graph.NewSequentialKey(0)
}
return graph.NewSequentialKey(horizon)
}
func (qs *TripleStore) AddTriple(t quad.Quad) {
batch := &leveldb.Batch{}
qs.buildWrite(batch, t)
err := qs.db.Write(batch, qs.writeopts)
if err != nil {
glog.Errorf("Couldn't write to DB for triple %s.", t)
return
}
qs.size++
}
func (ts *TripleStore) AddTriple(t *graph.Triple) {
batch := &leveldb.Batch{}
ts.buildWrite(batch, t)
err := ts.db.Write(batch, ts.writeopts)
if err != nil {
glog.Errorf("Couldn't write to DB for triple %s", t)
return
}
ts.size++
}
func (qs *QuadStore) buildQuadWrite(tx *bolt.Tx, q quad.Quad, id int64, isAdd bool) error {
var entry IndexEntry
b := tx.Bucket(spoBucket)
b.FillPercent = localFillPercent
data := b.Get(qs.createKeyFor(spo, q))
if data != nil {
// We got something.
err := json.Unmarshal(data, &entry)
if err != nil {
return err
}
}
if isAdd && len(entry.History)%2 == 1 {
glog.Errorf("attempt to add existing quad %v: %#v", entry, q)
return graph.ErrQuadExists
}
if !isAdd && len(entry.History)%2 == 0 {
glog.Errorf("attempt to delete non-existent quad %v: %#v", entry, q)
return graph.ErrQuadNotExist
}
entry.History = append(entry.History, id)
jsonbytes, err := json.Marshal(entry)
if err != nil {
glog.Errorf("Couldn't write to buffer for entry %#v: %s", entry, err)
return err
}
for _, index := range [][4]quad.Direction{spo, osp, pos, cps} {
if index == cps && q.Get(quad.Label) == "" {
continue
}
b := tx.Bucket(bucketFor(index))
b.FillPercent = localFillPercent
err = b.Put(qs.createKeyFor(index, q), jsonbytes)
if err != nil {
return err
}
}
return nil
}
func (qs *QuadStore) Horizon() graph.PrimaryKey {
var log MongoLogEntry
err := qs.db.C("log").Find(nil).Sort("-LogID").One(&log)
if err != nil {
if err == mgo.ErrNotFound {
return graph.NewSequentialKey(0)
}
glog.Errorf("Could not get Horizon from Mongo: %v", err)
}
return graph.NewSequentialKey(log.LogID)
}
func (qs *TripleStore) Horizon() int64 {
var log MongoLogEntry
err := qs.db.C("log").Find(nil).Sort("-LogID").One(&log)
if err != nil {
if err == mgo.ErrNotFound {
return 0
}
glog.Errorf("Could not get Horizon from Mongo: %v", err)
}
return log.LogID
}
func (qs *QuadStore) buildQuadWrite(batch *leveldb.Batch, q quad.Quad, id int64, isAdd bool) error {
var entry IndexEntry
data, err := qs.db.Get(qs.createKeyFor(spo, q), qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Error("could not access DB to prepare index: ", err)
return err
}
if err == nil {
// We got something.
err = json.Unmarshal(data, &entry)
if err != nil {
return err
}
} else {
entry.Quad = q
}
if isAdd && len(entry.History)%2 == 1 {
glog.Errorf("attempt to add existing quad %v: %#v", entry, q)
return graph.ErrQuadExists
}
if !isAdd && len(entry.History)%2 == 0 {
glog.Error("attempt to delete non-existent quad %v: %#c", entry, q)
return graph.ErrQuadNotExist
}
entry.History = append(entry.History, id)
bytes, err := json.Marshal(entry)
if err != nil {
glog.Errorf("could not write to buffer for entry %#v: %s", entry, err)
return err
}
batch.Put(qs.createKeyFor(spo, q), bytes)
batch.Put(qs.createKeyFor(osp, q), bytes)
batch.Put(qs.createKeyFor(pos, q), bytes)
if q.Get(quad.Label) != "" {
batch.Put(qs.createKeyFor(cps, q), bytes)
}
return nil
}
func (qs *QuadStore) buildQuadWriteLMDB(tx *lmdb.Txn, q quad.Quad, id int64, isAdd bool) error {
var entry proto.HistoryEntry
dbi := qs.dbis[spoDB]
data, err := tx.Get(dbi, qs.createKeyFor(spo, q))
if err == nil {
// We got something.
err := entry.Unmarshal(data)
if err != nil {
return err
}
}
if isAdd && len(entry.History)%2 == 1 {
glog.Errorf("attempt to add existing quad %v: %#v", entry, q)
return graph.ErrQuadExists
}
if !isAdd && len(entry.History)%2 == 0 {
glog.Errorf("attempt to delete non-existent quad %v: %#v", entry, q)
return graph.ErrQuadNotExist
}
entry.History = append(entry.History, uint64(id))
bytes, err := entry.Marshal()
if err != nil {
glog.Errorf("Couldn't write to buffer for entry %#v: %s", entry, err)
return err
}
for _, index := range [][4]quad.Direction{spo, osp, pos, cps} {
if index == cps && q.Get(quad.Label) == "" {
continue
}
dbi = qs.dbis[dbFor(index)]
err = tx.Put(dbi, qs.createKeyFor(index, q), bytes, 0)
if err != nil {
return err
}
}
return nil
}
func (qs *TripleStore) NameOf(v graph.Value) string {
val, ok := qs.idCache.Get(v.(string))
if ok {
return val
}
var node MongoNode
err := qs.db.C("nodes").FindId(v.(string)).One(&node)
if err != nil {
glog.Errorf("Error: Couldn't retrieve node %s %v", v, err)
}
qs.idCache.Put(v.(string), node.Name)
return node.Name
}
func (qs *TripleStore) Quad(val graph.Value) *quad.Quad {
var bsonDoc bson.M
err := qs.db.C("triples").FindId(val.(string)).One(&bsonDoc)
if err != nil {
glog.Errorf("Error: Couldn't retrieve triple %s %v", val, err)
}
return &quad.Quad{
bsonDoc["Subject"].(string),
bsonDoc["Predicate"].(string),
bsonDoc["Object"].(string),
bsonDoc["Label"].(string),
}
}
func (ts *TripleStore) buildTripleWrite(batch *leveldb.Batch, t *graph.Triple) {
bytes, err := json.Marshal(*t)
if err != nil {
glog.Errorf("Couldn't write to buffer for triple %s\n %s\n", t, err)
return
}
batch.Put(ts.createKeyFor(spo, t), bytes)
batch.Put(ts.createKeyFor(osp, t), bytes)
batch.Put(ts.createKeyFor(pos, t), bytes)
if t.Get(graph.Provenance) != "" {
batch.Put(ts.createProvKeyFor(pso, t), bytes)
}
}