func createNewLevelDB(path string, _ graph.Options) error { opts := &opt.Options{} db, err := leveldb.OpenFile(path, opts) if err != nil { clog.Errorf("Error: could not create database: %v", err) return err } defer db.Close() qs := &QuadStore{} qs.db = db qs.writeopts = &opt.WriteOptions{ Sync: true, } qs.readopts = &opt.ReadOptions{} _, err = qs.db.Get([]byte(horizonKey), qs.readopts) if err != nil && err != leveldb.ErrNotFound { clog.Errorf("couldn't read from leveldb during init") return err } if err != leveldb.ErrNotFound { return graph.ErrDatabaseExists } // Write some metadata qs.Close() return nil }
func upgradeLevelDB(path string, opts graph.Options) error { db, err := leveldb.OpenFile(path, &opt.Options{}) defer db.Close() if err != nil { clog.Errorf("Error, couldn't open! %v", err) return err } version, err := getVersion(db) if err != nil { clog.Errorf("error: %v", err) return err } if version == latestDataVersion { fmt.Printf("Already at latest version: %d\n", latestDataVersion) return nil } if version > latestDataVersion { err := fmt.Errorf("Unknown data version: %d -- upgrade this tool", version) clog.Errorf("error: %v", err) return err } for i := version; i < latestDataVersion; i++ { err := migrateFunctions[i](db) if err != nil { return err } setVersion(db, i+1, nil) } return nil }
func (qs *QuadStore) Quad(k graph.Value) quad.Quad { var in proto.HistoryEntry b, err := qs.db.Get(k.(Token), qs.readopts) if err == leveldb.ErrNotFound { // No harm, no foul. return quad.Quad{} } else if err != nil { clog.Errorf("Error: could not get quad from DB. %v", err) return quad.Quad{} } err = in.Unmarshal(b) if err != nil { clog.Errorf("Error: could not reconstruct history. %v", err) return quad.Quad{} } b, err = qs.db.Get(createDeltaKeyFor(int64(in.History[len(in.History)-1])), qs.readopts) if err == leveldb.ErrNotFound { // No harm, no foul. return quad.Quad{} } else if err != nil { clog.Errorf("Error: could not get quad from DB. %v", err) return quad.Quad{} } var d proto.LogDelta err = d.Unmarshal(b) if err != nil { clog.Errorf("Error: could not reconstruct quad. %v", err) return quad.Quad{} } return d.Quad.ToNative() }
func (it *AllIterator) makeCursor() { var cursor *sql.Rows var err error if it.cursor != nil { it.cursor.Close() } if it.table == "quads" { cursor, err = it.qs.db.Query(`SELECT subject_hash, predicate_hash, object_hash, label_hash FROM quads;`) if err != nil { clog.Errorf("Couldn't get cursor from SQL database: %v", err) cursor = nil } } else { if clog.V(4) { clog.Infof("sql: getting node query") } cursor, err = it.qs.db.Query(`SELECT hash FROM nodes;`) if err != nil { clog.Errorf("Couldn't get cursor from SQL database: %v", err) cursor = nil } if clog.V(4) { clog.Infof("sql: got node query") } } it.cursor = cursor }
func (qs *QuadStore) WriteHorizonAndSize(tx *bolt.Tx) error { buf := new(bytes.Buffer) err := binary.Write(buf, binary.LittleEndian, qs.size) if err != nil { clog.Errorf("Couldn't convert size!") return err } b := tx.Bucket(metaBucket) b.FillPercent = localFillPercent werr := b.Put([]byte("size"), buf.Bytes()) if werr != nil { clog.Errorf("Couldn't write size!") return werr } buf.Reset() err = binary.Write(buf, binary.LittleEndian, qs.horizon) if err != nil { clog.Errorf("Couldn't convert horizon!") } werr = b.Put([]byte("horizon"), buf.Bytes()) if werr != nil { clog.Errorf("Couldn't write horizon!") return werr } return err }
func (it *Iterator) Contains(v graph.Value) bool { graph.ContainsLogIn(it, v) if it.isAll { // The result needs to be set, so when contains is called, the result can be retrieved it.result = v return graph.ContainsLogOut(it, v, true) } t := v.(*Token) if t == nil { clog.Errorf("Could not cast to token") return graph.ContainsLogOut(it, v, false) } if t.Kind == nodeKind { clog.Errorf("Contains does not work with node values") return graph.ContainsLogOut(it, v, false) } // Contains is for when you want to know that an iterator refers to a quad var offset int switch it.dir { case quad.Subject: offset = 0 case quad.Predicate: offset = (quad.HashSize * 2) case quad.Object: offset = (quad.HashSize * 2) * 2 case quad.Label: offset = (quad.HashSize * 2) * 3 } val := t.Hash[offset : offset+(quad.HashSize*2)] if val == it.hash { return graph.ContainsLogOut(it, v, true) } return graph.ContainsLogOut(it, v, false) }
func NewAllIterator(qs *QuadStore, kind string) *Iterator { if kind != nodeKind && kind != quadKind { clog.Errorf("Cannot create iterator for an unknown kind") return &Iterator{done: true} } if qs.context == nil { clog.Errorf("Cannot create iterator without a valid context") return &Iterator{done: true} } var size int64 if kind == nodeKind { size = qs.NodeSize() } else { size = qs.Size() } return &Iterator{ uid: iterator.NextUID(), qs: qs, size: size, dir: quad.Any, isAll: true, kind: kind, done: false, } }
func createSQLTables(addr string, options graph.Options) error { conn, err := connectSQLTables(addr, options) if err != nil { return err } defer conn.Close() tx, err := conn.Begin() if err != nil { clog.Errorf("Couldn't begin creation transaction: %s", err) return err } table, err := tx.Exec(nodesTableStatement) if err != nil { tx.Rollback() errd := err.(*pq.Error) if errd.Code == "42P07" { return graph.ErrDatabaseExists } clog.Errorf("Cannot create nodes table: %v", table) return err } table, err = tx.Exec(` CREATE TABLE quads ( horizon BIGSERIAL PRIMARY KEY, subject_hash BYTEA NOT NULL, predicate_hash BYTEA NOT NULL, object_hash BYTEA NOT NULL, label_hash BYTEA, id BIGINT, ts timestamp );`) if err != nil { tx.Rollback() errd := err.(*pq.Error) if errd.Code == "42P07" { return graph.ErrDatabaseExists } clog.Errorf("Cannot create quad table: %v", table) return err } factor, factorOk, err := options.IntKey("db_fill_factor") if !factorOk { factor = defaultFillFactor } spoIndexes := quadsSecondaryIndexes(factor) var index sql.Result index, err = tx.Exec(quadsUniqueIndex + quadsForeignIndex + spoIndexes) if err != nil { clog.Errorf("Cannot create indices: %v", index) tx.Rollback() return err } tx.Commit() return nil }
func createSQLTables(addr string, options graph.Options) error { conn, err := connectSQLTables(addr, options) if err != nil { return err } defer conn.Close() tx, err := conn.Begin() if err != nil { clog.Errorf("Couldn't begin creation transaction: %s", err) return err } quadTable, err := tx.Exec(` CREATE TABLE quads ( subject TEXT NOT NULL, predicate TEXT NOT NULL, object TEXT NOT NULL, label TEXT, horizon BIGSERIAL PRIMARY KEY, id BIGINT, ts timestamp, subject_hash TEXT NOT NULL, predicate_hash TEXT NOT NULL, object_hash TEXT NOT NULL, label_hash TEXT, UNIQUE(subject_hash, predicate_hash, object_hash, label_hash) );`) if err != nil { tx.Rollback() errd := err.(*pq.Error) if errd.Code == "42P07" { return graph.ErrDatabaseExists } clog.Errorf("Cannot create quad table: %v", quadTable) return err } factor, factorOk, err := options.IntKey("db_fill_factor") if !factorOk { factor = 50 } var index sql.Result index, err = tx.Exec(fmt.Sprintf(` CREATE INDEX spo_index ON quads (subject_hash) WITH (FILLFACTOR = %d); CREATE INDEX pos_index ON quads (predicate_hash) WITH (FILLFACTOR = %d); CREATE INDEX osp_index ON quads (object_hash) WITH (FILLFACTOR = %d); `, factor, factor, factor)) if err != nil { clog.Errorf("Cannot create indices: %v", index) tx.Rollback() return err } tx.Commit() return nil }
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) { var qs QuadStore var err error qs.path = path cacheSize := DefaultCacheSize val, ok, err := options.IntKey("cache_size_mb") if err != nil { return nil, err } else if ok { cacheSize = val } qs.dbOpts = &opt.Options{ BlockCacheCapacity: cacheSize * opt.MiB, } qs.dbOpts.ErrorIfMissing = true writeBufferSize := DefaultWriteBufferSize val, ok, err = options.IntKey("writeBufferSize") if err != nil { return nil, err } else if ok { writeBufferSize = val } qs.dbOpts.WriteBuffer = writeBufferSize * opt.MiB qs.writeopts = &opt.WriteOptions{ Sync: false, } qs.readopts = &opt.ReadOptions{} db, err := leveldb.OpenFile(qs.path, qs.dbOpts) if err != nil { clog.Errorf("Error, could not open! %v", err) return nil, err } qs.db = db if clog.V(1) { clog.Infof("%v", qs.GetStats()) } vers, err := getVersion(qs.db) if err != nil { clog.Errorf("Error, could not read version info! %v", err) db.Close() return nil, err } else if vers != latestDataVersion { db.Close() return nil, fmt.Errorf("leveldb: data version is out of date (%d vs %d). Run cayleyupgrade for your config to update the data.", vers, latestDataVersion) } err = qs.getMetadata() if err != nil { db.Close() return nil, err } return &qs, nil }
func (it *SQLIterator) Contains(v graph.Value) bool { var err error if ok, res := it.sql.quickContains(v); ok { return res } err = it.makeCursor(false, v) if err != nil { clog.Errorf("Couldn't make query: %v", err) it.err = err if it.cursor != nil { it.cursor.Close() } return false } it.cols, err = it.cursor.Columns() if err != nil { clog.Errorf("Couldn't get columns") it.err = err it.cursor.Close() return false } it.resultList = nil for { if !it.cursor.Next() { if clog.V(4) { clog.Infof("sql: No next") } err := it.cursor.Err() if err != nil { clog.Errorf("Cursor error in SQL: %v", err) it.err = err } it.cursor.Close() break } s, err := scan(it.cursor, len(it.cols)) if err != nil { it.err = err it.cursor.Close() return false } it.resultList = append(it.resultList, s) } it.cursor.Close() it.cursor = nil if len(it.resultList) != 0 { it.resultIndex = 0 it.buildResult(0) return true } return false }
func toQuadValue(v value) quad.Value { if v == nil { return nil } switch d := v.(type) { case string: return quad.Raw(d) // compatibility case int64: return quad.Int(d) case float64: return quad.Float(d) case bool: return quad.Bool(d) case time.Time: return quad.Time(d) case bson.M: // TODO(dennwc): use raw document instead? so, ok := d["val"] if !ok { clog.Errorf("Error: Empty value in map: %v", v) return nil } s := so.(string) if len(d) == 1 { return quad.String(s) } if o, ok := d["iri"]; ok && o.(bool) { return quad.IRI(s) } else if o, ok := d["bnode"]; ok && o.(bool) { return quad.BNode(s) } else if o, ok := d["lang"]; ok && o.(string) != "" { return quad.LangString{ Value: quad.String(s), Lang: o.(string), } } else if o, ok := d["type"]; ok && o.(string) != "" { return quad.TypedString{ Value: quad.String(s), Type: quad.IRI(o.(string)), } } return quad.String(s) case []byte: var p proto.Value if err := p.Unmarshal(d); err != nil { clog.Errorf("Error: Couldn't decode value: %v", err) return nil } return p.ToNative() default: panic(fmt.Errorf("unsupported type: %T", v)) } }
func (qs *QuadStore) runTxPostgres(tx *sql.Tx, in []graph.Delta, opts graph.IgnoreOpts) error { allAdds := true for _, d := range in { if d.Action != graph.Add { allAdds = false } } if allAdds { return qs.copyFrom(tx, in) } for _, d := range in { switch d.Action { case graph.Add: _, err := tx.Exec(`INSERT INTO quads(subject, predicate, object, label, id, ts, subject_hash, predicate_hash, object_hash, label_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);`, d.Quad.Subject, d.Quad.Predicate, d.Quad.Object, d.Quad.Label, d.ID.Int(), d.Timestamp, hashOf(d.Quad.Subject), hashOf(d.Quad.Predicate), hashOf(d.Quad.Object), hashOf(d.Quad.Label), ) if err != nil { clog.Errorf("couldn't exec INSERT statement: %v", err) return err } case graph.Delete: result, err := tx.Exec(`DELETE FROM quads WHERE subject_hash=$1 and predicate_hash=$2 and object_hash=$3 and label_hash=$4;`, hashOf(d.Quad.Subject), hashOf(d.Quad.Predicate), hashOf(d.Quad.Object), hashOf(d.Quad.Label)) if err != nil { clog.Errorf("couldn't exec DELETE statement: %v", err) return err } affected, err := result.RowsAffected() if err != nil { clog.Errorf("couldn't get DELETE RowsAffected: %v", err) return err } if affected != 1 && !opts.IgnoreMissing { return errors.New("deleting non-existent triple; rolling back") } default: panic("unknown action") } } return nil }
func createNewBolt(path string, _ graph.Options) error { db, err := bolt.Open(path, 0600, nil) if err != nil { clog.Errorf("Error: couldn't create Bolt database: %v", err) return err } defer db.Close() qs := &QuadStore{} qs.db = db defer qs.Close() err = qs.getMetadata() if err != errNoBucket { return graph.ErrDatabaseExists } err = qs.createBuckets() if err != nil { return err } err = setVersion(qs.db, latestDataVersion) if err != nil { return err } qs.Close() return nil }
// Web stuff func (s *Session) Collate(result interface{}) { data, ok := result.(*Result) if !ok { clog.Errorf("unexpected result type: %T", result) return } if !data.metaresult { if data.val == nil { obj := make(map[string]interface{}) tags := data.actualResults var tagKeys []string for k := range tags { tagKeys = append(tagKeys, k) } sort.Strings(tagKeys) for _, k := range tagKeys { if name := s.qs.NameOf(tags[k]); name != nil { obj[k] = quadValueToNative(name) } else { delete(obj, k) } } if len(obj) != 0 { s.dataOutput = append(s.dataOutput, obj) } } else { s.dataOutput = append(s.dataOutput, data.val) } } }
// Next is a convenience function that conditionally calls the Next method // of an Iterator if it is a Nexter. If the Iterator is not a Nexter, Next // returns false. func Next(it Iterator) bool { if n, ok := it.(Nexter); ok { return n.Next() } clog.Errorf("Nexting an un-nextable iterator: %T", it) return false }
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) { var qs QuadStore var err error db, err := bolt.Open(path, 0600, nil) if err != nil { clog.Errorf("Error, couldn't open! %v", err) return nil, err } qs.db = db // BoolKey returns false on non-existence. IE, Sync by default. qs.db.NoSync, _, err = options.BoolKey("nosync") if err != nil { return nil, err } err = qs.getMetadata() if err == errNoBucket { return nil, errors.New("bolt: quadstore has not been initialised") } else if err != nil { return nil, err } if qs.version != latestDataVersion { return nil, errors.New("bolt: data version is out of date. Run cayleyupgrade for your config to update the data.") } return &qs, nil }
func (qs *QuadStore) copyFrom(tx *sql.Tx, in []graph.Delta) error { stmt, err := tx.Prepare(pq.CopyIn("quads", "subject", "predicate", "object", "label", "id", "ts", "subject_hash", "predicate_hash", "object_hash", "label_hash")) if err != nil { return err } for _, d := range in { _, err := stmt.Exec( d.Quad.Subject, d.Quad.Predicate, d.Quad.Object, d.Quad.Label, d.ID.Int(), d.Timestamp, hashOf(d.Quad.Subject), hashOf(d.Quad.Predicate), hashOf(d.Quad.Object), hashOf(d.Quad.Label), ) if err != nil { clog.Errorf("couldn't prepare COPY statement: %v", err) return err } } _, err = stmt.Exec() if err != nil { return err } return stmt.Close() }
func (qs *QuadStore) Quad(k graph.Value) quad.Quad { var d proto.LogDelta tok := k.(*Token) err := qs.db.View(func(tx *bolt.Tx) error { b := tx.Bucket(tok.bucket) data := b.Get(tok.key) if data == nil { return nil } var in proto.HistoryEntry err := in.Unmarshal(data) if err != nil { return err } if len(in.History) == 0 { return nil } b = tx.Bucket(logBucket) data = b.Get(qs.createDeltaKeyFor(int64(in.History[len(in.History)-1]))) if data == nil { // No harm, no foul. return nil } return d.Unmarshal(data) }) if err != nil { clog.Errorf("Error getting quad: %v", err) return quad.Quad{} } return d.Quad.ToNative() }
func (p *pathObject) toValue(call otto.FunctionCall, withTags bool) otto.Value { it := p.buildIteratorTree() it.Tagger().Add(TopResultTag) const limit = 1 var ( val otto.Value err error ) if !withTags { array := p.wk.runIteratorToArrayNoTags(it, limit) if len(array) < 1 { return otto.NullValue() } val, err = call.Otto.ToValue(array[0]) } else { array := p.wk.runIteratorToArray(it, limit) if len(array) < 1 { return otto.NullValue() } val, err = call.Otto.ToValue(array[0]) } if err != nil { clog.Errorf("%v", err) return otto.NullValue() } return val }
func (wk *worker) toArrayFunc(env *otto.Otto, obj *otto.Object, withTags bool) func(otto.FunctionCall) otto.Value { return func(call otto.FunctionCall) otto.Value { it := buildIteratorTree(obj, wk.qs) it.Tagger().Add(TopResultTag) limit := -1 if len(call.ArgumentList) > 0 { limitParsed, _ := call.Argument(0).ToInteger() limit = int(limitParsed) } var val otto.Value var err error if !withTags { array := wk.runIteratorToArrayNoTags(it, limit) val, err = call.Otto.ToValue(array) } else { array := wk.runIteratorToArray(it, limit) val, err = call.Otto.ToValue(array) } if err != nil { clog.Errorf("%v", err) return otto.NullValue() } return val } }
func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *QuadStore) *Iterator { vb := value.(Token) p := make([]byte, 0, 2+quad.HashSize) p = append(p, []byte(prefix)...) p = append(p, []byte(vb[1:])...) opts := &opt.ReadOptions{ DontFillCache: true, } it := Iterator{ uid: iterator.NextUID(), nextPrefix: p, checkID: vb, dir: d, originalPrefix: prefix, ro: opts, iter: qs.db.NewIterator(nil, opts), open: true, qs: qs, } ok := it.iter.Seek(it.nextPrefix) if !ok { it.open = false it.iter.Release() clog.Errorf("Opening LevelDB iterator couldn't seek to location %v", it.nextPrefix) } return &it }
func connectSQLTables(addr string, _ graph.Options) (*sql.DB, error) { // TODO(barakmich): Parse options for more friendly addr, other SQLs. conn, err := sql.Open("postgres", addr) if err != nil { clog.Errorf("Couldn't open database at %s: %#v", addr, err) return nil, err } // "Open may just validate its arguments without creating a connection to the database." // "To verify that the data source name is valid, call Ping." // Source: http://golang.org/pkg/database/sql/#Open if err := conn.Ping(); err != nil { clog.Errorf("Couldn't open database at %s: %#v", addr, err) return nil, err } return conn, nil }
func (wk *worker) toValueFunc(env *otto.Otto, obj *otto.Object, withTags bool) func(otto.FunctionCall) otto.Value { return func(call otto.FunctionCall) otto.Value { it := buildIteratorTree(obj, wk.qs) it.Tagger().Add(TopResultTag) limit := 1 var val otto.Value var err error if !withTags { array := wk.runIteratorToArrayNoTags(it, limit) if len(array) < 1 { return otto.NullValue() } val, err = call.Otto.ToValue(array[0]) } else { array := wk.runIteratorToArray(it, limit) if len(array) < 1 { return otto.NullValue() } val, err = call.Otto.ToValue(array[0]) } if err != nil { clog.Errorf("%v", err) return otto.NullValue() } return val } }
func (it *Iterator) Next() bool { var result struct { ID string `bson:"_id"` Added []int64 `bson:"Added"` Deleted []int64 `bson:"Deleted"` } if it.iter == nil { it.iter = it.makeMongoIterator() } found := it.iter.Next(&result) if !found { err := it.iter.Err() if err != nil { it.err = err clog.Errorf("Error Nexting Iterator: %v", err) } return false } if it.collection == "quads" && len(result.Added) <= len(result.Deleted) { return it.Next() } if it.collection == "quads" { it.result = QuadHash(result.ID) } else { it.result = NodeHash(result.ID) } return true }
func (p *pathObject) toArray(call otto.FunctionCall, withTags bool) otto.Value { args := exportArgs(call.ArgumentList) if len(args) > 1 { return otto.NullValue() } limit := -1 if len(args) > 0 { limit = toInt(args[0]) } it := p.buildIteratorTree() it.Tagger().Add(TopResultTag) var ( val otto.Value err error ) if !withTags { array := p.wk.runIteratorToArrayNoTags(it, limit) val, err = call.Otto.ToValue(array) } else { array := p.wk.runIteratorToArray(it, limit) val, err = call.Otto.ToValue(array) } if err != nil { clog.Errorf("%v", err) return otto.NullValue() } return val }
func (qs *QuadStore) sizeForIterator(isAll bool, dir quad.Direction, hash NodeHash) int64 { var err error if isAll { return qs.Size() } if qs.noSizes { if dir == quad.Predicate { return (qs.Size() / 100) + 1 } return (qs.Size() / 1000) + 1 } if val, ok := qs.sizes.Get(hash.String() + string(dir.Prefix())); ok { return val.(int64) } var size int64 if clog.V(4) { clog.Infof("sql: getting size for select %s, %v", dir.String(), hash) } err = qs.db.QueryRow( fmt.Sprintf("SELECT count(*) FROM quads WHERE %s_hash = $1;", dir.String()), hash.toSQL()).Scan(&size) if err != nil { clog.Errorf("Error getting size from SQL database: %v", err) return 0 } qs.sizes.Put(hash.String()+string(dir.Prefix()), size) return size }
func getViaData(obj *otto.Object) (predicates []interface{}, tags []string, ok bool) { argList, _ := obj.Get("_gremlin_values") if argList.Class() != "GoArray" { clog.Errorf("How is arglist not an array? Return nothing. %v", argList.Class()) return nil, nil, false } argArray := argList.Object() lengthVal, _ := argArray.Get("length") length, _ := lengthVal.ToInteger() if length == 0 { predicates = []interface{}{} } else { zero, _ := argArray.Get("0") predicates = buildPathFromValue(zero) } if length >= 2 { one, _ := argArray.Get("1") if one.IsString() { tags = append(tags, one.String()) } else if one.Class() == "Array" { tags = stringsFrom(one.Object()) } } ok = true return }
func (qs *QuadStore) Quad(val graph.Value) quad.Quad { var q quad.Quad err := qs.db.C("quads").FindId(val.(string)).One(&q) if err != nil { clog.Errorf("Error: Couldn't retrieve quad %s %v", val, err) } return q }
func setVersion(db *bolt.DB, version int64) error { return db.Update(func(tx *bolt.Tx) error { buf := new(bytes.Buffer) err := binary.Write(buf, binary.LittleEndian, version) if err != nil { clog.Errorf("Couldn't convert version!") return err } b := tx.Bucket(metaBucket) werr := b.Put([]byte("version"), buf.Bytes()) if werr != nil { clog.Errorf("Couldn't write version!") return werr } return nil }) }