func (it *Iterator) Contains(v graph.Value) bool { graph.ContainsLogIn(it, v) if it.isAll { // The result needs to be set, so when contains is called, the result can be retrieved it.result = v return graph.ContainsLogOut(it, v, true) } t := v.(*Token) if t == nil { glog.Error("Could not cast to token") return graph.ContainsLogOut(it, v, false) } if t.Kind == nodeKind { glog.Error("Contains does not work with node values") return graph.ContainsLogOut(it, v, false) } // Contains is for when you want to know that an iterator refers to a quad var offset int switch it.dir { case quad.Subject: offset = 0 case quad.Predicate: offset = (hashSize * 2) case quad.Object: offset = (hashSize * 2) * 2 case quad.Label: offset = (hashSize * 2) * 3 } val := t.Hash[offset : offset+(hashSize*2)] if val == it.hash { return graph.ContainsLogOut(it, v, true) } return graph.ContainsLogOut(it, v, false) }
func NewAllIterator(qs *QuadStore, kind string) *Iterator { if kind != nodeKind && kind != quadKind { glog.Error("Cannot create iterator for an unknown kind") return &Iterator{done: true} } if qs.context == nil { glog.Error("Cannot create iterator without a valid context") return &Iterator{done: true} } var size int64 if kind == nodeKind { size = qs.NodeSize() } else { size = qs.Size() } return &Iterator{ uid: iterator.NextUID(), qs: qs, size: size, dir: quad.Any, isAll: true, kind: kind, done: false, } }
// WriteHorizonAndSize ?? func (qs *QuadStore) WriteHorizonAndSize(tx *lmdb.Txn) error { buf := new(bytes.Buffer) err := binary.Write(buf, binary.LittleEndian, qs.size) if err != nil { glog.Errorf("Couldn't convert size!") return err } werr := tx.Put(qs.metaDBI, []byte("size"), buf.Bytes(), 0) if werr != nil { glog.Error("Couldn't write size!") return werr } buf.Reset() err = binary.Write(buf, binary.LittleEndian, qs.horizon) if err != nil { glog.Errorf("Couldn't convert horizon!") } werr = tx.Put(qs.metaDBI, []byte("horizon"), buf.Bytes(), 0) if werr != nil { glog.Error("Couldn't write horizon!") return werr } return err }
func (qs *QuadStore) WriteHorizonAndSize(tx *bolt.Tx) error { buf := new(bytes.Buffer) err := binary.Write(buf, binary.LittleEndian, qs.size) if err != nil { glog.Errorf("Couldn't convert size!") return err } b := tx.Bucket(metaBucket) b.FillPercent = localFillPercent werr := b.Put([]byte("size"), buf.Bytes()) if werr != nil { glog.Error("Couldn't write size!") return werr } buf.Reset() err = binary.Write(buf, binary.LittleEndian, qs.horizon) if err != nil { glog.Errorf("Couldn't convert horizon!") } werr = b.Put([]byte("horizon"), buf.Bytes()) if werr != nil { glog.Error("Couldn't write horizon!") return werr } return err }
func (qs *QuadStore) Quad(k graph.Value) quad.Quad { var d graph.Delta tok := k.(*Token) err := qs.db.View(func(tx *bolt.Tx) error { b := tx.Bucket(tok.bucket) data := b.Get(tok.key) if data == nil { return nil } var in IndexEntry err := json.Unmarshal(data, &in) if err != nil { return err } if len(in.History) == 0 { return nil } b = tx.Bucket(logBucket) data = b.Get(qs.createDeltaKeyFor(in.History[len(in.History)-1])) if data == nil { // No harm, no foul. return nil } return json.Unmarshal(data, &d) }) if err != nil { glog.Error("Error getting quad: ", err) return quad.Quad{} } return d.Quad }
func toArrayFunc(env *otto.Otto, ses *Session, obj *otto.Object, withTags bool) func(otto.FunctionCall) otto.Value { return func(call otto.FunctionCall) otto.Value { it := buildIteratorTree(obj, ses.ts) it.Tagger().Add(TopResultTag) limit := -1 if len(call.ArgumentList) > 0 { limitParsed, _ := call.Argument(0).ToInteger() limit = int(limitParsed) } var val otto.Value var err error if !withTags { array := runIteratorToArrayNoTags(it, ses, limit) val, err = call.Otto.ToValue(array) } else { array := runIteratorToArray(it, ses, limit) val, err = call.Otto.ToValue(array) } if err != nil { glog.Error(err) return otto.NullValue() } return val } }
func (qs *TripleStore) RemoveTriple(t quad.Quad) { _, err := qs.db.Get(qs.createKeyFor(spo, t), qs.readopts) if err != nil && err != leveldb.ErrNotFound { glog.Error("Couldn't access DB to confirm deletion") return } if err == leveldb.ErrNotFound { // No such triple in the database, forget about it. return } batch := &leveldb.Batch{} batch.Delete(qs.createKeyFor(spo, t)) batch.Delete(qs.createKeyFor(osp, t)) batch.Delete(qs.createKeyFor(pos, t)) qs.UpdateValueKeyBy(t.Get(quad.Subject), -1, batch) qs.UpdateValueKeyBy(t.Get(quad.Predicate), -1, batch) qs.UpdateValueKeyBy(t.Get(quad.Object), -1, batch) if t.Get(quad.Label) != "" { batch.Delete(qs.createProvKeyFor(pso, t)) qs.UpdateValueKeyBy(t.Get(quad.Label), -1, batch) } err = qs.db.Write(batch, nil) if err != nil { glog.Errorf("Couldn't delete triple %s.", t) return } qs.size-- }
func toValueFunc(env *otto.Otto, ses *Session, obj *otto.Object, withTags bool) func(otto.FunctionCall) otto.Value { return func(call otto.FunctionCall) otto.Value { it := buildIteratorTree(obj, ses.ts) it.Tagger().Add(TopResultTag) limit := 1 var val otto.Value var err error if !withTags { array := runIteratorToArrayNoTags(it, ses, limit) if len(array) < 1 { return otto.NullValue() } val, err = call.Otto.ToValue(array[0]) } else { array := runIteratorToArray(it, ses, limit) if len(array) < 1 { return otto.NullValue() } val, err = call.Otto.ToValue(array[0]) } if err != nil { glog.Error(err) return otto.NullValue() } else { return val } } }
func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *QuadStore) *Iterator { vb := value.(Token) p := make([]byte, 0, 2+hashSize) p = append(p, []byte(prefix)...) p = append(p, []byte(vb[1:])...) opts := &opt.ReadOptions{ DontFillCache: true, } it := Iterator{ uid: iterator.NextUID(), nextPrefix: p, checkID: vb, dir: d, originalPrefix: prefix, ro: opts, iter: qs.db.NewIterator(nil, opts), open: true, qs: qs, } ok := it.iter.Seek(it.nextPrefix) if !ok { it.open = false it.iter.Release() glog.Error("Opening LevelDB iterator couldn't seek to location ", it.nextPrefix) } return &it }
func (ts *TripleStore) Size() int64 { count, err := ts.db.C("triples").Count() if err != nil { glog.Error("Error: ", err) return 0 } return int64(count) }
func (qs *QuadStore) Quad(k graph.Value) quad.Quad { var q quad.Quad b, err := qs.db.Get(k.(Token), qs.readopts) if err != nil && err != leveldb.ErrNotFound { glog.Error("Error: could not get quad from DB.") return quad.Quad{} } if err == leveldb.ErrNotFound { // No harm, no foul. return quad.Quad{} } err = json.Unmarshal(b, &q) if err != nil { glog.Error("Error: could not reconstruct quad.") return quad.Quad{} } return q }
func (qs *TripleStore) Quad(k graph.Value) quad.Quad { var triple quad.Quad b, err := qs.db.Get(k.(Token), qs.readopts) if err != nil && err != leveldb.ErrNotFound { glog.Error("Error: couldn't get triple from DB.") return quad.Quad{} } if err == leveldb.ErrNotFound { // No harm, no foul. return quad.Quad{} } err = json.Unmarshal(b, &triple) if err != nil { glog.Error("Error: couldn't reconstruct triple.") return quad.Quad{} } return triple }
func (it *AllIterator) Next() bool { if it.done { return false } if len(it.buffer) <= it.offset+1 { it.offset = 0 var last []byte if it.buffer != nil { last = it.buffer[len(it.buffer)-1] } it.buffer = make([][]byte, 0, bufferSize) err := it.qs.db.View(func(tx *bolt.Tx) error { i := 0 b := tx.Bucket(it.bucket) cur := b.Cursor() if last == nil { k, _ := cur.First() var out []byte out = make([]byte, len(k)) copy(out, k) it.buffer = append(it.buffer, out) i++ } else { k, _ := cur.Seek(last) if !bytes.Equal(k, last) { return fmt.Errorf("could not pick up after %v", k) } } for i < bufferSize { k, _ := cur.Next() if k == nil { it.buffer = append(it.buffer, k) break } var out []byte out = make([]byte, len(k)) copy(out, k) it.buffer = append(it.buffer, out) i++ } return nil }) if err != nil { glog.Error("Error nexting in database: ", err) it.err = err it.done = true return false } } else { it.offset++ } if it.Result() == nil { it.done = true return false } return true }
func (qs *QuadStore) ApplyDeltas(deltas []graph.Delta) error { oldSize := qs.size oldHorizon := qs.horizon err := qs.db.Update(func(tx *bolt.Tx) error { b := tx.Bucket(logBucket) b.FillPercent = localFillPercent resizeMap := make(map[string]int64) sizeChange := int64(0) for _, d := range deltas { bytes, err := json.Marshal(d) if err != nil { return err } err = b.Put(qs.createDeltaKeyFor(d.ID), bytes) if err != nil { return err } } for _, d := range deltas { err := qs.buildQuadWrite(tx, d.Quad, d.ID, d.Action == graph.Add) if err != nil { return err } delta := int64(1) if d.Action == graph.Delete { delta = int64(-1) } resizeMap[d.Quad.Subject] += delta resizeMap[d.Quad.Predicate] += delta resizeMap[d.Quad.Object] += delta if d.Quad.Label != "" { resizeMap[d.Quad.Label] += delta } sizeChange += delta qs.horizon = d.ID } for k, v := range resizeMap { if v != 0 { err := qs.UpdateValueKeyBy(k, v, tx) if err != nil { return err } } } qs.size += sizeChange return qs.WriteHorizonAndSize(tx) }) if err != nil { glog.Error("Couldn't write to DB for Delta set. Error: ", err) qs.horizon = oldHorizon qs.size = oldSize return err } return nil }
func (qs *QuadStore) buildQuadWrite(tx *bolt.Tx, q quad.Quad, id int64, isAdd bool) error { var entry IndexEntry b := tx.Bucket(spoBucket) b.FillPercent = localFillPercent data := b.Get(qs.createKeyFor(spo, q)) if data != nil { // We got something. err := json.Unmarshal(data, &entry) if err != nil { return err } } if isAdd && len(entry.History)%2 == 1 { glog.Error("Adding a valid quad ", entry) return graph.ErrQuadExists } if !isAdd && len(entry.History)%2 == 0 { glog.Error("Deleting an invalid quad ", entry) return graph.ErrQuadNotExist } entry.History = append(entry.History, id) jsonbytes, err := json.Marshal(entry) if err != nil { glog.Errorf("Couldn't write to buffer for entry %#v: %s", entry, err) return err } for _, index := range [][4]quad.Direction{spo, osp, pos, cps} { if index == cps && q.Get(quad.Label) == "" { continue } b := tx.Bucket(bucketFor(index)) b.FillPercent = localFillPercent err = b.Put(qs.createKeyFor(index, q), jsonbytes) if err != nil { return err } } return nil }
// Quad ?? func (qs *QuadStore) Quad(k graph.Value) quad.Quad { var d proto.LogDelta tok := k.(*Token) err := qs.env.View(func(tx *lmdb.Txn) (err error) { tx.RawRead = true dbi := qs.dbis[tok.db] data, _ := tx.Get(dbi, tok.key) if data == nil { return nil } var in proto.HistoryEntry err = in.Unmarshal(data) if err != nil { return err } if len(in.History) == 0 { return nil } data, _ = tx.Get(qs.logDBI, qs.createDeltaKeyFor(int64(in.History[len(in.History)-1]))) if data == nil { // No harm, no foul. return nil } return d.Unmarshal(data) }) if err != nil { glog.Error("Error getting quad: ", err) return quad.Quad{} } if d.Quad == nil { glog.Error("Unable to get quad: ", err) return quad.Quad{} } return quad.Quad{ d.Quad.Subject, d.Quad.Predicate, d.Quad.Object, d.Quad.Label, } }
func NewIterator(qs *QuadStore, k string, d quad.Direction, val graph.Value) *Iterator { t := val.(*Token) if t == nil { glog.Error("Token == nil") } if t.Kind != nodeKind { glog.Error("Cannot create an iterator from a non-node value") return &Iterator{done: true} } if k != nodeKind && k != quadKind { glog.Error("Cannot create iterator for unknown kind") return &Iterator{done: true} } if qs.context == nil { glog.Error("Cannot create iterator without a valid context") return &Iterator{done: true} } name := qs.NameOf(t) // The number of references to this node is held in the nodes entity key := qs.createKeyFromToken(t) foundNode := new(NodeEntry) err := datastore.Get(qs.context, key, foundNode) if err != nil && err != datastore.ErrNoSuchEntity { glog.Errorf("Error: %v", err) return &Iterator{done: true} } size := foundNode.Size return &Iterator{ uid: iterator.NextUID(), name: name, dir: d, qs: qs, size: size, isAll: false, kind: k, hash: t.Hash, done: false, } }
func (qs *QuadStore) buildQuadWrite(batch *leveldb.Batch, q quad.Quad, id int64, isAdd bool) error { var entry IndexEntry data, err := qs.db.Get(qs.createKeyFor(spo, q), qs.readopts) if err != nil && err != leveldb.ErrNotFound { glog.Error("could not access DB to prepare index: ", err) return err } if err == nil { // We got something. err = json.Unmarshal(data, &entry) if err != nil { return err } } else { entry.Quad = q } if isAdd && len(entry.History)%2 == 1 { glog.Errorf("attempt to add existing quad %v: %#v", entry, q) return graph.ErrQuadExists } if !isAdd && len(entry.History)%2 == 0 { glog.Error("attempt to delete non-existent quad %v: %#c", entry, q) return graph.ErrQuadNotExist } entry.History = append(entry.History, id) bytes, err := json.Marshal(entry) if err != nil { glog.Errorf("could not write to buffer for entry %#v: %s", entry, err) return err } batch.Put(qs.createKeyFor(spo, q), bytes) batch.Put(qs.createKeyFor(osp, q), bytes) batch.Put(qs.createKeyFor(pos, q), bytes) if q.Get(quad.Label) != "" { batch.Put(qs.createKeyFor(cps, q), bytes) } return nil }
func (qs *QuadStore) ApplyDeltas(deltas []graph.Delta, ignoreOpts graph.IgnoreOpts) error { batch := &leveldb.Batch{} resizeMap := make(map[string]int64) sizeChange := int64(0) for _, d := range deltas { if d.Action != graph.Add && d.Action != graph.Delete { return errors.New("leveldb: invalid action") } bytes, err := json.Marshal(d) if err != nil { return err } batch.Put(keyFor(d), bytes) err = qs.buildQuadWrite(batch, d.Quad, d.ID.Int(), d.Action == graph.Add) if err != nil { if err == graph.ErrQuadExists && ignoreOpts.IgnoreDup { continue } if err == graph.ErrQuadNotExist && ignoreOpts.IgnoreMissing { continue } return err } delta := int64(1) if d.Action == graph.Delete { delta = int64(-1) } resizeMap[d.Quad.Subject] += delta resizeMap[d.Quad.Predicate] += delta resizeMap[d.Quad.Object] += delta if d.Quad.Label != "" { resizeMap[d.Quad.Label] += delta } sizeChange += delta qs.horizon = d.ID.Int() } for k, v := range resizeMap { if v != 0 { err := qs.UpdateValueKeyBy(k, v, batch) if err != nil { return err } } } err := qs.db.Write(batch, qs.writeopts) if err != nil { glog.Error("could not write to DB for quadset.") return err } qs.size += sizeChange return nil }
func (ts *TripleStore) writeTriple(t *graph.Triple) bool { tripledoc := bson.M{"_id": ts.getIdForTriple(t), "Sub": t.Sub, "Pred": t.Pred, "Obj": t.Obj, "Provenance": t.Provenance} err := ts.db.C("triples").Insert(tripledoc) if err != nil { // Among the reasons I hate MongoDB. "Errors don't happen! Right guys?" if err.(*mgo.LastError).Code == 11000 { return false } glog.Error("Error: ", err) return false } return true }
func (qs *TripleStore) Close() { buf := new(bytes.Buffer) err := binary.Write(buf, binary.LittleEndian, qs.size) if err == nil { werr := qs.db.Put([]byte("__size"), buf.Bytes(), qs.writeopts) if werr != nil { glog.Error("Couldn't write size before closing!") } } else { glog.Errorf("Couldn't convert size before closing!") } qs.db.Close() qs.open = false }
func reverseGremlinChainTo(env *otto.Otto, prevObj *otto.Object, tag string) (*otto.Object, *otto.Object) { env.Run("var _base_object = {}") base, err := env.Object("_base_object") if err != nil { glog.Error(err) return otto.NullValue().Object(), otto.NullValue().Object() } if isVertexChain(prevObj) { base.Set("_gremlin_type", "vertex") } else { base.Set("_gremlin_type", "morphism") } return reverseGremlinChainHelper(env, prevObj, base, tag) }
func (ts *TripleStore) updateNodeBy(node_name string, inc int) { var size MongoNode node := ts.ValueOf(node_name) err := ts.db.C("nodes").FindId(node).One(&size) if err != nil { if err.Error() == "not found" { // Not found. Okay. size.Id = node.(string) size.Name = node_name size.Size = inc } else { glog.Error("Error:", err) return } } else { size.Id = node.(string) size.Name = node_name size.Size += inc } // Removing something... if inc < 0 { if size.Size <= 0 { err := ts.db.C("nodes").RemoveId(node) if err != nil { glog.Error("Error: ", err, " while removing node ", node_name) return } } } _, err2 := ts.db.C("nodes").UpsertId(node, size) if err2 != nil { glog.Error("Error: ", err) } }
func (qs *TripleStore) buildQuadWrite(batch *leveldb.Batch, q quad.Quad, id int64, isAdd bool) error { var entry IndexEntry data, err := qs.db.Get(qs.createKeyFor(spo, q), qs.readopts) if err != nil && err != leveldb.ErrNotFound { glog.Error("Couldn't access DB to prepare index: ", err) return err } if err == nil { // We got something. err = json.Unmarshal(data, &entry) if err != nil { return err } } else { entry.Quad = q } entry.History = append(entry.History, id) if isAdd && len(entry.History)%2 == 0 { glog.Error("Entry History is out of sync for", entry) return errors.New("Odd index history") } bytes, err := json.Marshal(entry) if err != nil { glog.Errorf("Couldn't write to buffer for entry %#v: %s", entry, err) return err } batch.Put(qs.createKeyFor(spo, q), bytes) batch.Put(qs.createKeyFor(osp, q), bytes) batch.Put(qs.createKeyFor(pos, q), bytes) if q.Get(quad.Label) != "" { batch.Put(qs.createKeyFor(cps, q), bytes) } return nil }
func (qs *QuadStore) Close() { buf := new(bytes.Buffer) err := binary.Write(buf, binary.LittleEndian, qs.size) if err == nil { werr := qs.db.Put([]byte(sizeKey), buf.Bytes(), qs.writeopts) if werr != nil { glog.Error("could not write size before closing!") } } else { glog.Errorf("could not convert size before closing!") } buf.Reset() err = binary.Write(buf, binary.LittleEndian, qs.horizon) if err == nil { werr := qs.db.Put([]byte(horizonKey), buf.Bytes(), qs.writeopts) if werr != nil { glog.Error("could not write horizon before closing!") } } else { glog.Errorf("could not convert horizon before closing!") } qs.db.Close() qs.open = false }
func setVersionLMDB(env *lmdb.Env, metadbi lmdb.DBI, version int64) error { return env.Update(func(tx *lmdb.Txn) error { buf := new(bytes.Buffer) err := binary.Write(buf, binary.LittleEndian, version) if err != nil { glog.Errorf("Couldn't convert version!") return err } werr := tx.Put(metadbi, []byte("version"), buf.Bytes(), 0) if werr != nil { glog.Error("Couldn't write version!") return werr } return nil }) }
func newWorker(qs graph.QuadStore) *worker { env := otto.New() wk := &worker{ qs: qs, env: env, limit: -1, } graph, _ := env.Object("graph = {}") env.Run("g = graph") graph.Set("Vertex", func(call otto.FunctionCall) otto.Value { call.Otto.Run("var out = {}") out, err := call.Otto.Object("out") if err != nil { glog.Error(err.Error()) return otto.TrueValue() } out.Set("_gremlin_type", "vertex") args := argsOf(call) if len(args) > 0 { out.Set("string_args", args) } wk.embedTraversals(env, out) wk.embedFinals(env, out) return out.Value() }) env.Run("graph.V = graph.Vertex") graph.Set("Morphism", func(call otto.FunctionCall) otto.Value { call.Otto.Run("var out = {}") out, _ := call.Otto.Object("out") out.Set("_gremlin_type", "morphism") wk.embedTraversals(env, out) return out.Value() }) env.Run("graph.M = graph.Morphism") graph.Set("Emit", func(call otto.FunctionCall) otto.Value { value := call.Argument(0) if value.IsDefined() { wk.send(&Result{val: &value}) } return otto.NullValue() }) return wk }
func setVersion(db *bolt.DB, version int64) error { return db.Update(func(tx *bolt.Tx) error { buf := new(bytes.Buffer) err := binary.Write(buf, binary.LittleEndian, version) if err != nil { glog.Errorf("Couldn't convert version!") return err } b := tx.Bucket(metaBucket) werr := b.Put([]byte("version"), buf.Bytes()) if werr != nil { glog.Error("Couldn't write version!") return werr } return nil }) }
func (qs *TripleStore) ApplyDeltas(deltas []graph.Delta) error { batch := &leveldb.Batch{} resizeMap := make(map[string]int64) size_change := int64(0) for _, d := range deltas { bytes, err := json.Marshal(d) if err != nil { return err } batch.Put(qs.createDeltaKeyFor(d), bytes) err = qs.buildQuadWrite(batch, d.Quad, d.ID, d.Action == graph.Add) if err != nil { return err } delta := int64(1) if d.Action == graph.Delete { delta = int64(-1) } resizeMap[d.Quad.Subject] += delta resizeMap[d.Quad.Predicate] += delta resizeMap[d.Quad.Object] += delta if d.Quad.Label != "" { resizeMap[d.Quad.Label] += delta } size_change += delta qs.horizon = d.ID } for k, v := range resizeMap { if v != 0 { err := qs.UpdateValueKeyBy(k, v, batch) if err != nil { return err } } } err := qs.db.Write(batch, qs.writeopts) if err != nil { glog.Error("Couldn't write to DB for tripleset.") return err } qs.size += size_change return nil }
func NewIterator(bucket []byte, d quad.Direction, value graph.Value, qs *QuadStore) *Iterator { tok := value.(*Token) if !bytes.Equal(tok.bucket, nodeBucket) { glog.Error("creating an iterator from a non-node value") return &Iterator{done: true} } it := Iterator{ uid: iterator.NextUID(), bucket: bucket, dir: d, qs: qs, size: qs.SizeOf(value), } it.checkID = make([]byte, len(tok.key)) copy(it.checkID, tok.key) return &it }