func (qs *QuadStore) optimizeAndIterator(it *iterator.And) (graph.Iterator, bool) {
// Fail fast if nothing can happen
glog.V(4).Infoln("Entering optimizeAndIterator", it.UID())
found := false
for _, it := range it.SubIterators() {
glog.V(4).Infoln(it.Type())
if it.Type() == mongoType {
found = true
}
}
if !found {
glog.V(4).Infoln("Aborting optimizeAndIterator")
return it, false
}
newAnd := iterator.NewAnd(qs)
var mongoIt *Iterator
for _, it := range it.SubIterators() {
switch it.Type() {
case mongoType:
if mongoIt == nil {
mongoIt = it.(*Iterator)
} else {
newAnd.AddSubIterator(it)
}
case graph.LinksTo:
continue
default:
newAnd.AddSubIterator(it)
}
}
stats := mongoIt.Stats()
lset := []graph.Linkage{
{
Dir: mongoIt.dir,
Value: qs.ValueOf(mongoIt.name),
},
}
n := 0
for _, it := range it.SubIterators() {
if it.Type() == graph.LinksTo {
lto := it.(*iterator.LinksTo)
// Is it more effective to do the replacement, or let the mongo check the linksto?
ltostats := lto.Stats()
if (ltostats.ContainsCost+stats.NextCost)*stats.Size > (ltostats.NextCost+stats.ContainsCost)*ltostats.Size {
continue
}
newLto := NewLinksTo(qs, lto.SubIterators()[0], "quads", lto.Direction(), lset)
newAnd.AddSubIterator(newLto)
n++
}
}
if n == 0 {
return it, false
}
return newAnd.Optimize()
}
func (it *AllIterator) makeCursor() {
var cursor *sql.Rows
var err error
if it.cursor != nil {
it.cursor.Close()
}
if it.table == "quads" {
cursor, err = it.qs.db.Query(`SELECT subject, predicate, object, label FROM quads;`)
if err != nil {
glog.Errorln("Couldn't get cursor from SQL database: %v", err)
cursor = nil
}
} else {
glog.V(4).Infoln("sql: getting node query")
cursor, err = it.qs.db.Query(`SELECT node FROM
(
SELECT subject FROM quads
UNION
SELECT predicate FROM quads
UNION
SELECT object FROM quads
UNION
SELECT label FROM quads
) AS DistinctNodes (node) WHERE node IS NOT NULL;`)
if err != nil {
glog.Errorln("Couldn't get cursor from SQL database: %v", err)
cursor = nil
}
glog.V(4).Infoln("sql: got node query")
}
it.cursor = cursor
}
func (m *MqlSession) ExecInput(input string, c chan interface{}, limit int) {
defer close(c)
var mqlQuery interface{}
err := json.Unmarshal([]byte(input), &mqlQuery)
if err != nil {
return
}
m.currentQuery = NewMqlQuery(m)
m.currentQuery.BuildIteratorTree(mqlQuery)
if m.currentQuery.isError {
return
}
it, _ := m.currentQuery.it.Optimize()
if glog.V(2) {
glog.V(2).Infoln(it.DebugString(0))
}
for {
_, ok := it.Next()
if !ok {
break
}
tags := make(map[string]graph.TSVal)
it.TagResults(&tags)
c <- &tags
for it.NextResult() == true {
tags := make(map[string]graph.TSVal)
it.TagResults(&tags)
c <- &tags
}
}
}
func (s *Session) ExecInput(input string, c chan interface{}, _ int) {
defer close(c)
var mqlQuery interface{}
err := json.Unmarshal([]byte(input), &mqlQuery)
if err != nil {
return
}
s.currentQuery = NewQuery(s)
s.currentQuery.BuildIteratorTree(mqlQuery)
if s.currentQuery.isError() {
return
}
it, _ := s.currentQuery.it.Optimize()
if glog.V(2) {
glog.V(2).Infoln(it.DebugString(0))
}
for graph.Next(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
c <- tags
for it.NextPath() == true {
tags := make(map[string]graph.Value)
it.TagResults(tags)
c <- tags
}
}
}
func Load(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error {
block := make([]quad.Quad, 0, cfg.LoadSize)
count := 0
for {
t, err := dec.Unmarshal()
if err != nil {
if err == io.EOF {
break
}
return err
}
block = append(block, t)
if len(block) == cap(block) {
count += len(block)
err := qw.AddQuadSet(block)
if err != nil {
return fmt.Errorf("db: failed to load data: %v", err)
}
block = block[:0]
if glog.V(2) {
glog.V(2).Infof("Wrote %d quads.", count)
}
}
}
count += len(block)
err := qw.AddQuadSet(block)
if err != nil {
return fmt.Errorf("db: failed to load data: %v", err)
}
if glog.V(2) {
glog.V(2).Infof("Wrote %d quads.", count)
}
return nil
}
// Size is the number of values stored, if we've got them all.
// Otherwise, guess based on the size of the subiterator.
func (it *Materialize) Size() (int64, bool) {
if it.hasRun && !it.aborted {
glog.V(2).Infoln("returning size", it.actualSize)
return it.actualSize, true
}
glog.V(2).Infoln("bailing size", it.actualSize)
return it.subIt.Size()
}
func (l *SQLLinkIterator) buildSQL(next bool, val graph.Value) (string, []string) {
query := "SELECT "
t := []string{
fmt.Sprintf("%s.subject", l.tableName),
fmt.Sprintf("%s.predicate", l.tableName),
fmt.Sprintf("%s.object", l.tableName),
fmt.Sprintf("%s.label", l.tableName),
}
for _, v := range l.getTags() {
t = append(t, v.String())
}
query += strings.Join(t, ", ")
query += " FROM "
t = []string{}
var values []string
for _, k := range l.getTables() {
values = append(values, k.values...)
t = append(t, fmt.Sprintf("%s as %s", k.table, k.name))
}
query += strings.Join(t, ", ")
constraint, wherevalues := l.buildWhere()
if constraint != "" {
query += " WHERE "
}
values = append(values, wherevalues...)
if !next {
v := val.(quad.Quad)
if constraint != "" {
constraint += " AND "
} else {
constraint += " WHERE "
}
t = []string{
fmt.Sprintf("%s.subject_hash = ?", l.tableName),
fmt.Sprintf("%s.predicate_hash = ?", l.tableName),
fmt.Sprintf("%s.object_hash = ?", l.tableName),
fmt.Sprintf("%s.label_hash = ?", l.tableName),
}
constraint += strings.Join(t, " AND ")
values = append(values, hashOf(v.Subject))
values = append(values, hashOf(v.Predicate))
values = append(values, hashOf(v.Object))
values = append(values, hashOf(v.Label))
}
query += constraint
query += ";"
if glog.V(4) {
dstr := query
for i := 1; i <= len(values); i++ {
dstr = strings.Replace(dstr, "?", fmt.Sprintf("'%s'", values[i-1]), 1)
}
glog.V(4).Infoln(dstr)
}
return query, values
}
func ContainsLogOut(it Iterator, val Value, good bool) bool {
if glog.V(4) {
if good {
glog.V(4).Infof("%s %d CHECK CONTAINS %d GOOD", strings.ToUpper(it.Type().String()), it.UID(), val)
} else {
glog.V(4).Infof("%s %d CHECK CONTAINS %d BAD", strings.ToUpper(it.Type().String()), it.UID(), val)
}
}
return good
}
func NextLogOut(it Iterator, val TSVal, ok bool) (TSVal, bool) {
if glog.V(4) {
if ok {
glog.V(4).Infof("%s %d NEXT IS %d", strings.ToUpper(it.Type()), it.GetUid(), val)
} else {
glog.V(4).Infof("%s %d NEXT DONE", strings.ToUpper(it.Type()), it.GetUid())
}
}
return val, ok
}
func NextLogOut(it Iterator, val Value, ok bool) (Value, bool) {
if glog.V(4) {
if ok {
glog.V(4).Infof("%s %d NEXT IS %d", strings.ToUpper(it.Type().String()), it.UID(), val)
} else {
glog.V(4).Infof("%s %d NEXT DONE", strings.ToUpper(it.Type().String()), it.UID())
}
}
return val, ok
}
func CheckLogOut(it Iterator, val TSVal, good bool) bool {
if glog.V(4) {
if good {
glog.V(4).Infof("%s %d CHECK %d GOOD", strings.ToUpper(it.Type()), it.GetUid(), val)
} else {
glog.V(4).Infof("%s %d CHECK %d BAD", strings.ToUpper(it.Type()), it.GetUid(), val)
}
}
return good
}
// Check a value against our internal iterator. In order to do this, we must first open a new
// iterator of "triples that have `val` in our direction", given to us by the triple store,
// and then Next() values out of that iterator and Check() them against our subiterator.
func (it *HasA) Check(val graph.TSVal) bool {
graph.CheckLogIn(it, val)
if glog.V(4) {
glog.V(4).Infoln("Id is", it.ts.GetNameFor(val))
}
// TODO(barakmich): Optimize this
if it.resultIt != nil {
it.resultIt.Close()
}
it.resultIt = it.ts.GetTripleIterator(it.dir, val)
return graph.CheckLogOut(it, val, it.GetCheckResult())
}
// Check a value against our internal iterator. In order to do this, we must first open a new
// iterator of "triples that have `val` in our direction", given to us by the triple store,
// and then Next() values out of that iterator and Contains() them against our subiterator.
func (it *HasA) Contains(val graph.Value) bool {
graph.ContainsLogIn(it, val)
if glog.V(4) {
glog.V(4).Infoln("Id is", it.ts.NameOf(val))
}
// TODO(barakmich): Optimize this
if it.resultIt != nil {
it.resultIt.Close()
}
it.resultIt = it.ts.TripleIterator(it.dir, val)
return graph.ContainsLogOut(it, val, it.NextContains())
}
// Check a value against our internal iterator. In order to do this, we must first open a new
// iterator of "triples that have `val` in our direction", given to us by the triple store,
// and then Next() values out of that iterator and Check() them against our subiterator.
func (h *HasaIterator) Check(val TSVal) bool {
CheckLogIn(h, val)
if glog.V(4) {
glog.V(4).Infoln("Id is", h.ts.GetNameFor(val))
}
// TODO(barakmich): Optimize this
if h.resultIt != nil {
h.resultIt.Close()
}
h.resultIt = h.ts.GetTripleIterator(h.direction, val)
return CheckLogOut(h, val, h.GetCheckResult())
}
// NextContains() is shared code between Contains() and GetNextResult() -- calls next on the
// result iterator (a triple iterator based on the last checked value) and returns true if
// another match is made.
func (it *HasA) NextContains() bool {
for graph.Next(it.resultIt) {
link := it.resultIt.Result()
if glog.V(4) {
glog.V(4).Infoln("Quad is", it.ts.Quad(link))
}
if it.primaryIt.Contains(link) {
it.result = it.ts.TripleDirection(link, it.dir)
return true
}
}
return false
}
// NextContains() is shared code between Contains() and GetNextResult() -- calls next on the
// result iterator (a quad iterator based on the last checked value) and returns true if
// another match is made.
func (it *HasA) NextContains() bool {
for graph.Next(it.resultIt) {
it.runstats.ContainsNext += 1
link := it.resultIt.Result()
if glog.V(4) {
glog.V(4).Infoln("Quad is", it.qs.Quad(link))
}
if it.primaryIt.Contains(link) {
it.result = it.qs.QuadDirection(link, it.dir)
return true
}
}
it.err = it.resultIt.Err()
return false
}
// Get the next result that matches this branch.
func (it *HasA) NextPath() bool {
// Order here is important. If the subiterator has a NextPath, then we
// need do nothing -- there is a next result, and we shouldn't move forward.
// However, we then need to get the next result from our last Contains().
//
// The upshot is, the end of NextPath() bubbles up from the bottom of the
// iterator tree up, and we need to respect that.
glog.V(4).Infoln("HASA", it.UID(), "NextPath")
if it.primaryIt.NextPath() {
return true
}
result := it.NextContains()
glog.V(4).Infoln("HASA", it.UID(), "NextPath Returns", result, "")
return result
}
// Called by subclases.
func BaseInit(it *Base) {
// Your basic iterator is nextable
it.canNext = true
if glog.V(2) {
it.uid = nextID()
}
}
func (ts *TripleStore) NameOf(k graph.Value) string {
if k == nil {
glog.V(2).Infoln("k was nil")
return ""
}
return ts.valueData(k.([]byte)).Name
}
func (qs *TripleStore) NameOf(k graph.Value) string {
if k == nil {
glog.V(2).Info("k was nil")
return ""
}
return qs.valueData(k.(Token)).Name
}
func (qs *QuadStore) sizeForIterator(isAll bool, dir quad.Direction, val string) int64 {
var err error
if isAll {
return qs.Size()
}
if qs.noSizes {
if dir == quad.Predicate {
return (qs.Size() / 100) + 1
}
return (qs.Size() / 1000) + 1
}
if val, ok := qs.lru.Get(val + string(dir.Prefix())); ok {
return val
}
var size int64
glog.V(4).Infoln("sql: getting size for select %s, %s", dir.String(), val)
err = qs.db.QueryRow(
fmt.Sprintf("SELECT count(*) FROM quads WHERE %s_hash = $1;", dir.String()), hashOf(val)).Scan(&size)
if err != nil {
glog.Errorln("Error getting size from SQL database: %v", err)
return 0
}
qs.lru.Put(val+string(dir.Prefix()), size)
return size
}
func (ts *TripleStore) GetNameFor(k graph.TSVal) string {
if k == nil {
glog.V(2).Infoln("k was nil")
return ""
}
return ts.getValueData(k.([]byte)).Name
}
// GetCheckResult() is shared code between Check() and GetNextResult() -- calls next on the
// result iterator (a triple iterator based on the last checked value) and returns true if
// another match is made.
func (h *HasaIterator) GetCheckResult() bool {
for {
linkVal, ok := h.resultIt.Next()
if !ok {
break
}
if glog.V(4) {
glog.V(4).Infoln("Triple is", h.ts.GetTriple(linkVal).ToString())
}
if h.primaryIt.Check(linkVal) {
h.Last = h.ts.GetTripleDirection(linkVal, h.direction)
return true
}
}
return false
}
// NextContains() is shared code between Contains() and GetNextResult() -- calls next on the
// result iterator (a triple iterator based on the last checked value) and returns true if
// another match is made.
func (it *HasA) NextContains() bool {
for {
linkVal, ok := graph.Next(it.resultIt)
if !ok {
break
}
if glog.V(4) {
glog.V(4).Infoln("Quad is", it.ts.Quad(linkVal))
}
if it.primaryIt.Contains(linkVal) {
it.result = it.ts.TripleDirection(linkVal, it.dir)
return true
}
}
return false
}
func (s *Session) ExecInput(input string, c chan interface{}, _ int) {
defer close(c)
var mqlQuery interface{}
err := json.Unmarshal([]byte(input), &mqlQuery)
if err != nil {
return
}
s.currentQuery = NewQuery(s)
s.currentQuery.BuildIteratorTree(mqlQuery)
if s.currentQuery.isError() {
return
}
it, _ := s.currentQuery.it.Optimize()
if glog.V(2) {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
glog.Infof("failed to format description: %v", err)
} else {
glog.Infof("%s", b)
}
}
for graph.Next(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
c <- tags
for it.NextPath() == true {
tags := make(map[string]graph.Value)
it.TagResults(tags)
c <- tags
}
}
}
// GetCheckResult() is shared code between Check() and GetNextResult() -- calls next on the
// result iterator (a triple iterator based on the last checked value) and returns true if
// another match is made.
func (it *HasA) GetCheckResult() bool {
for {
linkVal, ok := it.resultIt.Next()
if !ok {
break
}
if glog.V(4) {
glog.V(4).Infoln("Triple is", it.ts.GetTriple(linkVal))
}
if it.primaryIt.Check(linkVal) {
it.Last = it.ts.GetTripleDirection(linkVal, it.dir)
return true
}
}
return false
}
func (qs *QuadStore) NameOf(v graph.Value) string {
if v == nil {
glog.V(2).Info("NameOf was nil")
return ""
}
return v.(string)
}
// NameOf ??
func (qs *QuadStore) NameOf(k graph.Value) string {
if k == nil {
glog.V(2).Info("k was nil")
return ""
}
return qs.valueDataLMDB(k.(*Token)).Name
}
func (wk *worker) runIterator(it graph.Iterator) {
if wk.wantShape() {
iterator.OutputQueryShapeForIterator(it, wk.qs, wk.shape)
return
}
it, _ = it.Optimize()
if glog.V(2) {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
glog.Infof("failed to format description: %v", err)
} else {
glog.Infof("%s", b)
}
}
for {
select {
case <-wk.kill:
return
default:
}
if !graph.Next(it) {
break
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
if !wk.send(&Result{actualResults: tags}) {
break
}
for it.NextPath() {
select {
case <-wk.kill:
return
default:
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
if !wk.send(&Result{actualResults: tags}) {
break
}
}
}
if glog.V(2) {
bytes, _ := json.MarshalIndent(graph.DumpStats(it), "", " ")
glog.V(2).Infoln(string(bytes))
}
it.Close()
}
// Check a value against our internal iterator. In order to do this, we must first open a new
// iterator of "quads that have `val` in our direction", given to us by the quad store,
// and then Next() values out of that iterator and Contains() them against our subiterator.
func (it *HasA) Contains(val graph.Value) bool {
graph.ContainsLogIn(it, val)
it.runstats.Contains += 1
if glog.V(4) {
glog.V(4).Infoln("Id is", it.qs.NameOf(val))
}
// TODO(barakmich): Optimize this
if it.resultIt != nil {
it.resultIt.Close()
}
it.resultIt = it.qs.QuadIterator(it.dir, val)
ok := it.NextContains()
if it.err != nil {
return false
}
return graph.ContainsLogOut(it, val, ok)
}