func TestMultipleConstraintParse(t *testing.T) {
qs, _ := graph.NewQuadStore("memstore", "", nil)
w, _ := graph.NewQuadWriter("single", qs, nil)
for _, tv := range []quad.Quad{
{"i", "like", "food", ""},
{"i", "like", "beer", ""},
{"you", "like", "beer", ""},
} {
w.AddQuad(tv)
}
query := `(
$a
(:like :beer)
(:like "food")
)`
it := BuildIteratorTreeForQuery(qs, query)
if it.Type() != graph.And {
t.Errorf("Odd iterator tree. Got: %#v", it.Describe())
}
if !graph.Next(it) {
t.Error("Got no results")
}
out := it.Result()
if out != qs.ValueOf("i") {
t.Errorf("Got %d, expected %d", out, qs.ValueOf("i"))
}
if graph.Next(it) {
t.Error("Too many results")
}
}
// Next()ing a LinksTo operates as described above.
func (it *LinksTo) Next() bool {
graph.NextLogIn(it)
it.runstats.Next += 1
if graph.Next(it.nextIt) {
it.runstats.ContainsNext += 1
it.result = it.nextIt.Result()
return graph.NextLogOut(it, it.nextIt, true)
}
// If there's an error in the 'next' iterator, we save it and we're done.
it.err = it.nextIt.Err()
if it.err != nil {
return false
}
// Subiterator is empty, get another one
if !graph.Next(it.primaryIt) {
// Possibly save error
it.err = it.primaryIt.Err()
// We're out of nodes in our subiterator, so we're done as well.
return graph.NextLogOut(it, 0, false)
}
it.nextIt.Close()
it.nextIt = it.qs.QuadIterator(it.dir, it.primaryIt.Result())
// Recurse -- return the first in the next set.
return it.Next()
}
func (s *Session) Execute(input string, c chan interface{}, _ int) {
defer close(c)
var mqlQuery interface{}
err := json.Unmarshal([]byte(input), &mqlQuery)
if err != nil {
return
}
s.currentQuery = NewQuery(s)
s.currentQuery.BuildIteratorTree(mqlQuery)
if s.currentQuery.isError() {
return
}
it, _ := s.currentQuery.it.Optimize()
if glog.V(2) {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
glog.Infof("failed to format description: %v", err)
} else {
glog.Infof("%s", b)
}
}
for graph.Next(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
c <- tags
for it.NextPath() == true {
tags := make(map[string]graph.Value)
it.TagResults(tags)
c <- tags
}
}
}
func TestRemoveQuad(t *testing.T) {
qs, w, _ := makeTestStore(simpleGraph)
w.RemoveQuad(quad.Quad{
Subject: "E",
Predicate: "follows",
Object: "F",
Label: "",
})
fixed := qs.FixedIterator()
fixed.Add(qs.ValueOf("E"))
fixed2 := qs.FixedIterator()
fixed2.Add(qs.ValueOf("follows"))
innerAnd := iterator.NewAnd(qs)
innerAnd.AddSubIterator(iterator.NewLinksTo(qs, fixed, quad.Subject))
innerAnd.AddSubIterator(iterator.NewLinksTo(qs, fixed2, quad.Predicate))
hasa := iterator.NewHasA(qs, innerAnd, quad.Object)
newIt, _ := hasa.Optimize()
if graph.Next(newIt) {
t.Error("E should not have any followers.")
}
}
func (qs *QuadStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bool) {
subs := it.SubIterators()
if len(subs) != 1 {
return it, false
}
primary := subs[0]
if primary.Type() == graph.Fixed {
size, _ := primary.Size()
if size == 1 {
if !graph.Next(primary) {
panic("unexpected size during optimize")
}
val := primary.Result()
newIt := qs.QuadIterator(it.Direction(), val)
nt := newIt.Tagger()
nt.CopyFrom(it)
for _, tag := range primary.Tagger().Tags() {
nt.AddFixed(tag, val)
}
it.Close()
return newIt, true
}
}
return it, false
}
// Next advances the Or graph.iterator. Because the Or is the union of its
// subiterators, it must produce from all subiterators -- unless it it
// shortcircuiting, in which case, it is the first one that returns anything.
func (it *Or) Next() bool {
graph.NextLogIn(it)
var first bool
for {
if it.currentIterator == -1 {
it.currentIterator = 0
first = true
}
curIt := it.internalIterators[it.currentIterator]
if graph.Next(curIt) {
it.result = curIt.Result()
return graph.NextLogOut(it, it.result, true)
}
it.err = curIt.Err()
if it.err != nil {
return graph.NextLogOut(it, nil, false)
}
if it.isShortCircuiting && !first {
break
}
it.currentIterator++
if it.currentIterator == it.itCount {
break
}
}
return graph.NextLogOut(it, nil, false)
}
func iterated(it graph.Iterator) []int {
var res []int
for graph.Next(it) {
res = append(res, it.Result().(int))
}
return res
}
func TestOptimize(t *testing.T) {
tmpDir, _ := ioutil.TempDir(os.TempDir(), "cayley_test")
t.Log(tmpDir)
defer os.RemoveAll(tmpDir)
err := createNewLevelDB(tmpDir, nil)
if err != nil {
t.Fatalf("Failed to create working directory")
}
qs, err := newQuadStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb QuadStore.")
}
w, _ := writer.NewSingleReplication(qs, nil)
w.AddQuadSet(makeQuadSet())
// With an linksto-fixed pair
fixed := qs.FixedIterator()
fixed.Add(qs.ValueOf("F"))
fixed.Tagger().Add("internal")
lto := iterator.NewLinksTo(qs, fixed, quad.Object)
oldIt := lto.Clone()
newIt, ok := lto.Optimize()
if !ok {
t.Errorf("Failed to optimize iterator")
}
if newIt.Type() != Type() {
t.Errorf("Optimized iterator type does not match original, got:%v expect:%v", newIt.Type(), Type())
}
newQuads := iteratedQuads(qs, newIt)
oldQuads := iteratedQuads(qs, oldIt)
if !reflect.DeepEqual(newQuads, oldQuads) {
t.Errorf("Optimized iteration does not match original")
}
graph.Next(oldIt)
oldResults := make(map[string]graph.Value)
oldIt.TagResults(oldResults)
graph.Next(newIt)
newResults := make(map[string]graph.Value)
newIt.TagResults(newResults)
if !reflect.DeepEqual(newResults, oldResults) {
t.Errorf("Discordant tag results, new:%v old:%v", newResults, oldResults)
}
}
func iteratedNames(qs graph.QuadStore, it graph.Iterator) []string {
var res []string
for graph.Next(it) {
res = append(res, qs.NameOf(it.Result()))
}
sort.Strings(res)
return res
}
func iteratedQuads(qs graph.QuadStore, it graph.Iterator) []quad.Quad {
var res ordered
for graph.Next(it) {
res = append(res, qs.Quad(it.Result()))
}
sort.Sort(res)
return res
}
func runTopLevel(path *Path) []string {
var out []string
it := path.BuildIterator()
it, _ = it.Optimize()
for graph.Next(it) {
v := path.qs.NameOf(it.Result())
out = append(out, v)
}
return out
}
func (it *Comparison) Next() bool {
for graph.Next(it.subIt) {
val := it.subIt.Result()
if it.doComparison(val) {
it.result = val
return true
}
}
it.err = it.subIt.Err()
return false
}
// Returns advances the And iterator. Because the And is the intersection of its
// subiterators, it must choose one subiterator to produce a candidate, and check
// this value against the subiterators. A productive choice of primary iterator
// is therefore very important.
func (it *And) Next() bool {
graph.NextLogIn(it)
it.runstats.Next += 1
for graph.Next(it.primaryIt) {
curr := it.primaryIt.Result()
if it.subItsContain(curr, nil) {
it.result = curr
return graph.NextLogOut(it, curr, true)
}
}
it.err = it.primaryIt.Err()
return graph.NextLogOut(it, nil, false)
}
// Next advances the Not iterator. It returns whether there is another valid
// new value. It fetches the next value of the all iterator which is not
// contained by the primary iterator.
func (it *Not) Next() bool {
graph.NextLogIn(it)
it.runstats.Next += 1
for graph.Next(it.allIt) {
if curr := it.allIt.Result(); !it.primaryIt.Contains(curr) {
it.result = curr
it.runstats.ContainsNext += 1
return graph.NextLogOut(it, curr, true)
}
}
it.err = it.allIt.Err()
return graph.NextLogOut(it, nil, false)
}
func iterateResults(qs graph.QuadStore, it graph.Iterator) []string {
var res []string
for graph.Next(it) {
v := it.Result()
if t, ok := v.(*Token); ok && t.Kind == nodeKind {
res = append(res, qs.NameOf(it.Result()))
} else {
res = append(res, qs.Quad(it.Result()).String())
}
}
sort.Strings(res)
it.Reset()
return res
}
// Next advances the subiterator, continuing until it returns a value which it
// has not previously seen.
func (it *Unique) Next() bool {
graph.NextLogIn(it)
it.runstats.Next += 1
for graph.Next(it.subIt) {
curr := it.subIt.Result()
if ok := it.seen[curr]; !ok {
it.result = curr
it.seen[curr] = true
return graph.NextLogOut(it, it.result, true)
}
}
it.err = it.subIt.Err()
return graph.NextLogOut(it, nil, false)
}
// NextContains() is shared code between Contains() and GetNextResult() -- calls next on the
// result iterator (a quad iterator based on the last checked value) and returns true if
// another match is made.
func (it *HasA) NextContains() bool {
for graph.Next(it.resultIt) {
it.runstats.ContainsNext += 1
link := it.resultIt.Result()
if glog.V(4) {
glog.V(4).Infoln("Quad is", it.qs.Quad(link))
}
if it.primaryIt.Contains(link) {
it.result = it.qs.QuadDirection(link, it.dir)
return true
}
}
it.err = it.resultIt.Err()
return false
}
func runTag(path *Path, tag string) []string {
var out []string
it := path.BuildIterator()
it, _ = it.Optimize()
for graph.Next(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
out = append(out, path.qs.NameOf(tags[tag]))
for it.NextPath() {
tags := make(map[string]graph.Value)
it.TagResults(tags)
out = append(out, path.qs.NameOf(tags[tag]))
}
}
return out
}
func (wk *worker) runIterator(it graph.Iterator) {
if wk.wantShape() {
iterator.OutputQueryShapeForIterator(it, wk.qs, wk.shape)
return
}
it, _ = it.Optimize()
if glog.V(2) {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
glog.Infof("failed to format description: %v", err)
} else {
glog.Infof("%s", b)
}
}
for {
select {
case <-wk.kill:
return
default:
}
if !graph.Next(it) {
break
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
if !wk.send(&Result{actualResults: tags}) {
break
}
for it.NextPath() {
select {
case <-wk.kill:
return
default:
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
if !wk.send(&Result{actualResults: tags}) {
break
}
}
}
if glog.V(2) {
bytes, _ := json.MarshalIndent(graph.DumpStats(it), "", " ")
glog.V(2).Infoln(string(bytes))
}
it.Close()
}
// Next advances the iterator. This is simpler than Contains. We have a
// subiterator we can get a value from, and we can take that resultant quad,
// pull our direction out of it, and return that.
func (it *HasA) Next() bool {
graph.NextLogIn(it)
it.runstats.Next += 1
if it.resultIt != nil {
it.resultIt.Close()
}
it.resultIt = &Null{}
if !graph.Next(it.primaryIt) {
it.err = it.primaryIt.Err()
return graph.NextLogOut(it, 0, false)
}
tID := it.primaryIt.Result()
val := it.qs.QuadDirection(tID, it.dir)
it.result = val
return graph.NextLogOut(it, val, true)
}
func (wk *worker) runIteratorWithCallback(it graph.Iterator, callback otto.Value, this otto.FunctionCall, limit int) {
n := 0
it, _ = it.Optimize()
if glog.V(2) {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
glog.V(2).Infof("failed to format description: %v", err)
} else {
glog.V(2).Infof("%s", b)
}
}
for {
select {
case <-wk.kill:
return
default:
}
if !graph.Next(it) {
break
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
val, _ = callback.Call(this.This, val)
n++
if limit >= 0 && n >= limit {
break
}
for it.NextPath() {
select {
case <-wk.kill:
return
default:
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
val, _ = callback.Call(this.This, val)
n++
if limit >= 0 && n >= limit {
break
}
}
}
it.Close()
}
func (it *Materialize) materializeSet() {
i := 0
for graph.Next(it.subIt) {
i++
if i > abortMaterializeAt {
it.aborted = true
break
}
id := it.subIt.Result()
val := id
if h, ok := id.(Keyer); ok {
val = h.Key()
}
if _, ok := it.containsMap[val]; !ok {
it.containsMap[val] = len(it.values)
it.values = append(it.values, nil)
}
index := it.containsMap[val]
tags := make(map[string]graph.Value)
it.subIt.TagResults(tags)
it.values[index] = append(it.values[index], result{id: id, tags: tags})
it.actualSize += 1
for it.subIt.NextPath() {
i++
if i > abortMaterializeAt {
it.aborted = true
break
}
tags := make(map[string]graph.Value)
it.subIt.TagResults(tags)
it.values[index] = append(it.values[index], result{id: id, tags: tags})
it.actualSize += 1
}
}
it.err = it.subIt.Err()
if it.err == nil && it.aborted {
if glog.V(2) {
glog.V(2).Infoln("Aborting subiterator")
}
it.values = nil
it.containsMap = nil
it.subIt.Reset()
}
it.hasRun = true
}
func TestTreeConstraintTagParse(t *testing.T) {
qs, _ := graph.NewQuadStore("memstore", "", nil)
w, _ := graph.NewQuadWriter("single", qs, nil)
w.AddQuad(quad.Quad{"i", "like", "food", ""})
w.AddQuad(quad.Quad{"food", "is", "good", ""})
query := "(\"i\"\n" +
"(:like\n" +
"($a (:is :good))))"
it := BuildIteratorTreeForQuery(qs, query)
if !graph.Next(it) {
t.Error("Got no results")
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
if qs.NameOf(tags["$a"]) != "food" {
t.Errorf("Got %s, expected food", qs.NameOf(tags["$a"]))
}
}
func TestTreeConstraintParse(t *testing.T) {
qs, _ := graph.NewQuadStore("memstore", "", nil)
w, _ := graph.NewQuadWriter("single", qs, nil)
w.AddQuad(quad.Quad{"i", "like", "food", ""})
w.AddQuad(quad.Quad{"food", "is", "good", ""})
query := "(\"i\"\n" +
"(:like\n" +
"($a (:is :good))))"
it := BuildIteratorTreeForQuery(qs, query)
if it.Type() != graph.And {
t.Errorf("Odd iterator tree. Got: %#v", it.Describe())
}
if !graph.Next(it) {
t.Error("Got no results")
}
out := it.Result()
if out != qs.ValueOf("i") {
t.Errorf("Got %d, expected %d", out, qs.ValueOf("i"))
}
}
func (wk *worker) runIteratorToArrayNoTags(it graph.Iterator, limit int) []string {
output := make([]string, 0)
n := 0
it, _ = it.Optimize()
for {
select {
case <-wk.kill:
return nil
default:
}
if !graph.Next(it) {
break
}
output = append(output, wk.qs.NameOf(it.Result()))
n++
if limit >= 0 && n >= limit {
break
}
}
it.Close()
return output
}
func (it *Materialize) Next() bool {
graph.NextLogIn(it)
it.runstats.Next += 1
if !it.hasRun {
it.materializeSet()
}
if it.err != nil {
return false
}
if it.aborted {
n := graph.Next(it.subIt)
it.err = it.subIt.Err()
return n
}
it.index++
it.subindex = 0
if it.index >= len(it.values) {
return graph.NextLogOut(it, nil, false)
}
return graph.NextLogOut(it, it.Result(), true)
}
func (wk *worker) runIteratorToArray(it graph.Iterator, limit int) []map[string]string {
output := make([]map[string]string, 0)
n := 0
it, _ = it.Optimize()
for {
select {
case <-wk.kill:
return nil
default:
}
if !graph.Next(it) {
break
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
output = append(output, wk.tagsToValueMap(tags))
n++
if limit >= 0 && n >= limit {
break
}
for it.NextPath() {
select {
case <-wk.kill:
return nil
default:
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
output = append(output, wk.tagsToValueMap(tags))
n++
if limit >= 0 && n >= limit {
break
}
}
}
it.Close()
return output
}
func TestMemstoreBackedSexp(t *testing.T) {
qs, _ := graph.NewQuadStore("memstore", "", nil)
w, _ := graph.NewQuadWriter("single", qs, nil)
emptyIt := BuildIteratorTreeForQuery(qs, "()")
if emptyIt.Type() != graph.Null {
t.Errorf(`Incorrect type for empty query, got:%q expect: "null"`, emptyIt.Type())
}
for _, test := range testQueries {
if test.add.IsValid() {
w.AddQuad(test.add)
}
it := BuildIteratorTreeForQuery(qs, test.query)
if it.Type() != test.typ {
t.Errorf("Incorrect type for %s, got:%q expect %q", test.message, it.Type(), test.expect)
}
if !graph.Next(it) {
t.Errorf("Failed to %s", test.message)
}
got := it.Result()
if expect := qs.ValueOf(test.expect); got != expect {
t.Errorf("Incorrect result for %s, got:%v expect %v", test.message, got, expect)
}
}
}
func (s *Session) Execute(input string, out chan interface{}, limit int) {
it := BuildIteratorTreeForQuery(s.qs, input)
newIt, changed := it.Optimize()
if changed {
it = newIt
}
if s.debug {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
fmt.Printf("failed to format description: %v", err)
} else {
fmt.Printf("%s", b)
}
}
nResults := 0
for graph.Next(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
out <- &tags
nResults++
if nResults > limit && limit != -1 {
break
}
for it.NextPath() == true {
tags := make(map[string]graph.Value)
it.TagResults(tags)
out <- &tags
nResults++
if nResults > limit && limit != -1 {
break
}
}
}
close(out)
}
func TestIterator(t *testing.T) {
tmpDir, err := ioutil.TempDir(os.TempDir(), "cayley_test")
if err != nil {
t.Fatalf("Could not create working directory: %v", err)
}
defer os.RemoveAll(tmpDir)
t.Log(tmpDir)
err = createNewLevelDB(tmpDir, nil)
if err != nil {
t.Fatal("Failed to create LevelDB database.")
}
qs, err := newQuadStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb QuadStore.")
}
w, _ := writer.NewSingleReplication(qs, nil)
w.AddQuadSet(makeQuadSet())
var it graph.Iterator
it = qs.NodesAllIterator()
if it == nil {
t.Fatal("Got nil iterator.")
}
size, exact := it.Size()
if size <= 0 || size >= 20 {
t.Errorf("Unexpected size, got:%d expect:(0, 20)", size)
}
if exact {
t.Errorf("Got unexpected exact result.")
}
if typ := it.Type(); typ != graph.All {
t.Errorf("Unexpected iterator type, got:%v expect:%v", typ, graph.All)
}
optIt, changed := it.Optimize()
if changed || optIt != it {
t.Errorf("Optimize unexpectedly changed iterator.")
}
expect := []string{
"A",
"B",
"C",
"D",
"E",
"F",
"G",
"follows",
"status",
"cool",
"status_graph",
}
sort.Strings(expect)
for i := 0; i < 2; i++ {
got := iteratedNames(qs, it)
sort.Strings(got)
if !reflect.DeepEqual(got, expect) {
t.Errorf("Unexpected iterated result on repeat %d, got:%v expect:%v", i, got, expect)
}
it.Reset()
}
for _, pq := range expect {
if !it.Contains(qs.ValueOf(pq)) {
t.Errorf("Failed to find and check %q correctly", pq)
}
}
// FIXME(kortschak) Why does this fail?
/*
for _, pq := range []string{"baller"} {
if it.Contains(qs.ValueOf(pq)) {
t.Errorf("Failed to check %q correctly", pq)
}
}
*/
it.Reset()
it = qs.QuadsAllIterator()
graph.Next(it)
q := qs.Quad(it.Result())
set := makeQuadSet()
var ok bool
for _, t := range set {
if t.String() == q.String() {
ok = true
break
}
}
if !ok {
t.Errorf("Failed to find %q during iteration, got:%q", q, set)
}
qs.Close()
}