func iterated(it graph.Iterator) []int {
var res []int
for graph.Next(it) {
res = append(res, it.Result().(int))
}
return res
}
func (qs *QuadStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {
switch it.Type() {
case graph.LinksTo:
return qs.optimizeLinksTo(it.(*iterator.LinksTo))
}
return it, false
}
func iteratedQuads(qs graph.QuadStore, it graph.Iterator) []quad.Quad {
var res ordered
for graph.Next(it) {
res = append(res, qs.Quad(it.Result()))
}
sort.Sort(res)
return res
}
func iteratedNames(qs graph.QuadStore, it graph.Iterator) []string {
var res []string
for graph.Next(it) {
res = append(res, qs.NameOf(it.Result()))
}
sort.Strings(res)
return res
}
func OutputQueryShapeForIterator(it graph.Iterator, qs graph.QuadStore, outputMap map[string]interface{}) {
s := &queryShape{
qs: qs,
nodeID: 1,
}
node := s.MakeNode(it.Clone())
s.AddNode(node)
outputMap["nodes"] = s.nodes
outputMap["links"] = s.links
}
// moveTagsTo() gets the tags for all of the src's subiterators and the
// src itself, and moves them to dst.
func moveTagsTo(dst graph.Iterator, src *And) {
tags := src.getSubTags()
for _, tag := range dst.Tagger().Tags() {
if _, ok := tags[tag]; ok {
delete(tags, tag)
}
}
dt := dst.Tagger()
for k := range tags {
dt.Add(k)
}
}
func TestIterators(t *testing.T) {
glog.Info("\n-----------\n")
inst, opts, err := createInstance()
defer inst.Close()
if err != nil {
t.Fatalf("failed to create instance: %v", err)
}
qs, _, _ := makeTestStore(simpleGraph, opts)
if qs.Size() != 11 {
t.Fatal("Incorrect number of quads")
}
var expected = []string{
quad.Quad{"C", "follows", "B", ""}.String(),
quad.Quad{"C", "follows", "D", ""}.String(),
}
it := qs.QuadIterator(quad.Subject, qs.ValueOf("C"))
if got, ok := compareResults(qs, it, expected); !ok {
t.Errorf("Unexpected iterated result, got:%v expect:%v", got, expected)
}
// Test contains
it = qs.QuadIterator(quad.Label, qs.ValueOf("status_graph"))
gqs := qs.(*QuadStore)
key := gqs.createKeyForQuad(quad.Quad{"G", "status", "cool", "status_graph"})
token := &Token{quadKind, key.StringID()}
if !it.Contains(token) {
t.Error("Contains failed")
}
// Test cloning an iterator
var it2 graph.Iterator
it2 = it.Clone()
x := it2.Describe()
y := it.Describe()
if x.Name != y.Name {
t.Errorf("Iterator Clone was not successful got: %v, expected: %v", x.Name, y.Name)
}
}
func iterateResults(qs graph.QuadStore, it graph.Iterator) []string {
var res []string
for graph.Next(it) {
v := it.Result()
if t, ok := v.(*Token); ok && t.Kind == nodeKind {
res = append(res, qs.NameOf(it.Result()))
} else {
res = append(res, qs.Quad(it.Result()).String())
}
}
sort.Strings(res)
it.Reset()
return res
}
func buildInOutIterator(obj *otto.Object, qs graph.QuadStore, base graph.Iterator, isReverse bool) graph.Iterator {
argList, _ := obj.Get("_gremlin_values")
if argList.Class() != "GoArray" {
glog.Errorln("How is arglist not an array? Return nothing.", argList.Class())
return iterator.NewNull()
}
argArray := argList.Object()
lengthVal, _ := argArray.Get("length")
length, _ := lengthVal.ToInteger()
var predicateNodeIterator graph.Iterator
if length == 0 {
predicateNodeIterator = qs.NodesAllIterator()
} else {
zero, _ := argArray.Get("0")
predicateNodeIterator = buildIteratorFromValue(zero, qs)
}
if length >= 2 {
var tags []string
one, _ := argArray.Get("1")
if one.IsString() {
tags = append(tags, one.String())
} else if one.Class() == "Array" {
tags = stringsFrom(one.Object())
}
for _, tag := range tags {
predicateNodeIterator.Tagger().Add(tag)
}
}
in, out := quad.Subject, quad.Object
if isReverse {
in, out = out, in
}
lto := iterator.NewLinksTo(qs, base, in)
and := iterator.NewAnd(qs)
and.AddSubIterator(iterator.NewLinksTo(qs, predicateNodeIterator, quad.Predicate))
and.AddSubIterator(lto)
return iterator.NewHasA(qs, and, out)
}
func (wk *worker) runIteratorToArray(it graph.Iterator, limit int) []map[string]string {
output := make([]map[string]string, 0)
n := 0
it, _ = it.Optimize()
for {
select {
case <-wk.kill:
return nil
default:
}
if !graph.Next(it) {
break
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
output = append(output, wk.tagsToValueMap(tags))
n++
if limit >= 0 && n >= limit {
break
}
for it.NextPath() {
select {
case <-wk.kill:
return nil
default:
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
output = append(output, wk.tagsToValueMap(tags))
n++
if limit >= 0 && n >= limit {
break
}
}
}
it.Close()
return output
}
func (wk *worker) runIteratorToArrayNoTags(it graph.Iterator, limit int) []string {
output := make([]string, 0)
n := 0
it, _ = it.Optimize()
for {
select {
case <-wk.kill:
return nil
default:
}
if !graph.Next(it) {
break
}
output = append(output, wk.qs.NameOf(it.Result()))
n++
if limit >= 0 && n >= limit {
break
}
}
it.Close()
return output
}
// optimizeOrder(l) takes a list and returns a list, containing the same contents
// but with a new ordering, however it wishes.
func (it *And) optimizeOrder(its []graph.Iterator) []graph.Iterator {
var (
// bad contains iterators that can't be (efficiently) nexted, such as
// graph.Optional or graph.Not. Separate them out and tack them on at the end.
out, bad []graph.Iterator
best graph.Iterator
bestCost = int64(1 << 62)
)
// Find the iterator with the projected "best" total cost.
// Total cost is defined as The Next()ed iterator's cost to Next() out
// all of it's contents, and to Contains() each of those against everyone
// else.
for _, root := range its {
if _, canNext := root.(graph.Nexter); !canNext {
bad = append(bad, root)
continue
}
rootStats := root.Stats()
cost := rootStats.NextCost
for _, f := range its {
if _, canNext := f.(graph.Nexter); !canNext {
continue
}
if f == root {
continue
}
stats := f.Stats()
cost += stats.ContainsCost * (1 + (rootStats.Size / (stats.Size + 1)))
}
cost *= rootStats.Size
if glog.V(3) {
glog.V(3).Infoln("And:", it.UID(), "Root:", root.UID(), "Total Cost:", cost, "Best:", bestCost)
}
if cost < bestCost {
best = root
bestCost = cost
}
}
if glog.V(3) {
glog.V(3).Infoln("And:", it.UID(), "Choosing:", best.UID(), "Best:", bestCost)
}
// TODO(barakmich): Optimization of order need not stop here. Picking a smart
// Contains() order based on probability of getting a false Contains() first is
// useful (fail faster).
// Put the best iterator (the one we wish to Next()) at the front...
out = append(out, best)
// ... push everyone else after...
for _, it := range its {
if _, canNext := it.(graph.Nexter); !canNext {
continue
}
if it != best {
out = append(out, it)
}
}
// ...and finally, the difficult children on the end.
return append(out, bad...)
}
func TestIterator(t *testing.T) {
tmpDir, err := ioutil.TempDir(os.TempDir(), "cayley_test")
if err != nil {
t.Fatalf("Could not create working directory: %v", err)
}
defer os.RemoveAll(tmpDir)
t.Log(tmpDir)
err = createNewLevelDB(tmpDir, nil)
if err != nil {
t.Fatal("Failed to create LevelDB database.")
}
qs, err := newQuadStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb QuadStore.")
}
w, _ := writer.NewSingleReplication(qs, nil)
w.AddQuadSet(makeQuadSet())
var it graph.Iterator
it = qs.NodesAllIterator()
if it == nil {
t.Fatal("Got nil iterator.")
}
size, exact := it.Size()
if size <= 0 || size >= 20 {
t.Errorf("Unexpected size, got:%d expect:(0, 20)", size)
}
if exact {
t.Errorf("Got unexpected exact result.")
}
if typ := it.Type(); typ != graph.All {
t.Errorf("Unexpected iterator type, got:%v expect:%v", typ, graph.All)
}
optIt, changed := it.Optimize()
if changed || optIt != it {
t.Errorf("Optimize unexpectedly changed iterator.")
}
expect := []string{
"A",
"B",
"C",
"D",
"E",
"F",
"G",
"follows",
"status",
"cool",
"status_graph",
}
sort.Strings(expect)
for i := 0; i < 2; i++ {
got := iteratedNames(qs, it)
sort.Strings(got)
if !reflect.DeepEqual(got, expect) {
t.Errorf("Unexpected iterated result on repeat %d, got:%v expect:%v", i, got, expect)
}
it.Reset()
}
for _, pq := range expect {
if !it.Contains(qs.ValueOf(pq)) {
t.Errorf("Failed to find and check %q correctly", pq)
}
}
// FIXME(kortschak) Why does this fail?
/*
for _, pq := range []string{"baller"} {
if it.Contains(qs.ValueOf(pq)) {
t.Errorf("Failed to check %q correctly", pq)
}
}
*/
it.Reset()
it = qs.QuadsAllIterator()
graph.Next(it)
q := qs.Quad(it.Result())
set := makeQuadSet()
var ok bool
for _, t := range set {
if t.String() == q.String() {
ok = true
break
}
}
if !ok {
t.Errorf("Failed to find %q during iteration, got:%q", q, set)
}
qs.Close()
}
func (wk *worker) runIteratorWithCallback(it graph.Iterator, callback otto.Value, this otto.FunctionCall, limit int) {
n := 0
it, _ = it.Optimize()
if glog.V(2) {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
glog.V(2).Infof("failed to format description: %v", err)
} else {
glog.V(2).Infof("%s", b)
}
}
for {
select {
case <-wk.kill:
return
default:
}
if !graph.Next(it) {
break
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
val, _ = callback.Call(this.This, val)
n++
if limit >= 0 && n >= limit {
break
}
for it.NextPath() {
select {
case <-wk.kill:
return
default:
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
val, _ = callback.Call(this.This, val)
n++
if limit >= 0 && n >= limit {
break
}
}
}
it.Close()
}
func (wk *worker) runIterator(it graph.Iterator) {
if wk.wantShape() {
iterator.OutputQueryShapeForIterator(it, wk.qs, wk.shape)
return
}
it, _ = it.Optimize()
if glog.V(2) {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
glog.Infof("failed to format description: %v", err)
} else {
glog.Infof("%s", b)
}
}
for {
select {
case <-wk.kill:
return
default:
}
if !graph.Next(it) {
break
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
if !wk.send(&Result{actualResults: tags}) {
break
}
for it.NextPath() {
select {
case <-wk.kill:
return
default:
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
if !wk.send(&Result{actualResults: tags}) {
break
}
}
}
if glog.V(2) {
bytes, _ := json.MarshalIndent(graph.DumpStats(it), "", " ")
glog.V(2).Infoln(string(bytes))
}
it.Close()
}
func (s *queryShape) MakeNode(it graph.Iterator) *Node {
n := Node{ID: s.nodeID}
for _, tag := range it.Tagger().Tags() {
n.Tags = append(n.Tags, tag)
}
for k := range it.Tagger().Fixed() {
n.Tags = append(n.Tags, k)
}
switch it.Type() {
case graph.And:
for _, sub := range it.SubIterators() {
s.nodeID++
newNode := s.MakeNode(sub)
if sub.Type() != graph.Or {
s.StealNode(&n, newNode)
} else {
s.AddNode(newNode)
s.AddLink(&Link{n.ID, newNode.ID, 0, 0})
}
}
case graph.Fixed:
n.IsFixed = true
for graph.Next(it) {
n.Values = append(n.Values, s.qs.NameOf(it.Result()))
}
case graph.HasA:
hasa := it.(*HasA)
s.PushHasa(n.ID, hasa.dir)
s.nodeID++
newNode := s.MakeNode(hasa.primaryIt)
s.AddNode(newNode)
s.RemoveHasa()
case graph.Or:
for _, sub := range it.SubIterators() {
s.nodeID++
newNode := s.MakeNode(sub)
if sub.Type() == graph.Or {
s.StealNode(&n, newNode)
} else {
s.AddNode(newNode)
s.AddLink(&Link{n.ID, newNode.ID, 0, 0})
}
}
case graph.LinksTo:
n.IsLinkNode = true
lto := it.(*LinksTo)
s.nodeID++
newNode := s.MakeNode(lto.primaryIt)
hasaID, hasaDir := s.LastHasa()
if (hasaDir == quad.Subject && lto.dir == quad.Object) ||
(hasaDir == quad.Object && lto.dir == quad.Subject) {
s.AddNode(newNode)
if hasaDir == quad.Subject {
s.AddLink(&Link{hasaID, newNode.ID, 0, n.ID})
} else {
s.AddLink(&Link{newNode.ID, hasaID, 0, n.ID})
}
} else if lto.primaryIt.Type() == graph.Fixed {
s.StealNode(&n, newNode)
} else {
s.AddNode(newNode)
}
case graph.Optional:
// Unsupported, for the moment
fallthrough
case graph.All:
}
return &n
}
func printIterator(qs graph.QuadStore, it graph.Iterator) {
for graph.Next(it) {
glog.Infof("%v", qs.Quad(it.Result()))
}
}
func buildIteratorTreeHelper(obj *otto.Object, qs graph.QuadStore, base graph.Iterator) graph.Iterator {
// TODO: Better error handling
var (
it graph.Iterator
subIt graph.Iterator
)
if prev, _ := obj.Get("_gremlin_prev"); !prev.IsObject() {
subIt = base
} else {
subIt = buildIteratorTreeHelper(prev.Object(), qs, base)
}
stringArgs := propertiesOf(obj, "string_args")
val, _ := obj.Get("_gremlin_type")
switch val.String() {
case "vertex":
if len(stringArgs) == 0 {
it = qs.NodesAllIterator()
} else {
fixed := qs.FixedIterator()
for _, name := range stringArgs {
fixed.Add(qs.ValueOf(name))
}
it = fixed
}
case "tag":
it = subIt
for _, tag := range stringArgs {
it.Tagger().Add(tag)
}
case "save":
all := qs.NodesAllIterator()
if len(stringArgs) > 2 || len(stringArgs) == 0 {
return iterator.NewNull()
}
if len(stringArgs) == 2 {
all.Tagger().Add(stringArgs[1])
} else {
all.Tagger().Add(stringArgs[0])
}
predFixed := qs.FixedIterator()
predFixed.Add(qs.ValueOf(stringArgs[0]))
subAnd := iterator.NewAnd(qs)
subAnd.AddSubIterator(iterator.NewLinksTo(qs, predFixed, quad.Predicate))
subAnd.AddSubIterator(iterator.NewLinksTo(qs, all, quad.Object))
hasa := iterator.NewHasA(qs, subAnd, quad.Subject)
and := iterator.NewAnd(qs)
and.AddSubIterator(hasa)
and.AddSubIterator(subIt)
it = and
case "saver":
all := qs.NodesAllIterator()
if len(stringArgs) > 2 || len(stringArgs) == 0 {
return iterator.NewNull()
}
if len(stringArgs) == 2 {
all.Tagger().Add(stringArgs[1])
} else {
all.Tagger().Add(stringArgs[0])
}
predFixed := qs.FixedIterator()
predFixed.Add(qs.ValueOf(stringArgs[0]))
subAnd := iterator.NewAnd(qs)
subAnd.AddSubIterator(iterator.NewLinksTo(qs, predFixed, quad.Predicate))
subAnd.AddSubIterator(iterator.NewLinksTo(qs, all, quad.Subject))
hasa := iterator.NewHasA(qs, subAnd, quad.Object)
and := iterator.NewAnd(qs)
and.AddSubIterator(hasa)
and.AddSubIterator(subIt)
it = and
case "has":
fixed := qs.FixedIterator()
if len(stringArgs) < 2 {
return iterator.NewNull()
}
for _, name := range stringArgs[1:] {
fixed.Add(qs.ValueOf(name))
}
predFixed := qs.FixedIterator()
predFixed.Add(qs.ValueOf(stringArgs[0]))
subAnd := iterator.NewAnd(qs)
subAnd.AddSubIterator(iterator.NewLinksTo(qs, predFixed, quad.Predicate))
subAnd.AddSubIterator(iterator.NewLinksTo(qs, fixed, quad.Object))
hasa := iterator.NewHasA(qs, subAnd, quad.Subject)
and := iterator.NewAnd(qs)
and.AddSubIterator(hasa)
and.AddSubIterator(subIt)
it = and
case "morphism":
it = base
case "and":
arg, _ := obj.Get("_gremlin_values")
firstArg, _ := arg.Object().Get("0")
if !isVertexChain(firstArg.Object()) {
return iterator.NewNull()
}
argIt := buildIteratorTree(firstArg.Object(), qs)
and := iterator.NewAnd(qs)
and.AddSubIterator(subIt)
and.AddSubIterator(argIt)
it = and
case "back":
arg, _ := obj.Get("_gremlin_back_chain")
argIt := buildIteratorTree(arg.Object(), qs)
and := iterator.NewAnd(qs)
and.AddSubIterator(subIt)
and.AddSubIterator(argIt)
it = and
case "is":
fixed := qs.FixedIterator()
for _, name := range stringArgs {
fixed.Add(qs.ValueOf(name))
}
and := iterator.NewAnd(qs)
and.AddSubIterator(fixed)
and.AddSubIterator(subIt)
it = and
case "or":
arg, _ := obj.Get("_gremlin_values")
firstArg, _ := arg.Object().Get("0")
if !isVertexChain(firstArg.Object()) {
return iterator.NewNull()
}
argIt := buildIteratorTree(firstArg.Object(), qs)
or := iterator.NewOr()
or.AddSubIterator(subIt)
or.AddSubIterator(argIt)
it = or
case "both":
// Hardly the most efficient pattern, but the most general.
// Worth looking into an Optimize() optimization here.
clone := subIt.Clone()
it1 := buildInOutIterator(obj, qs, subIt, false)
it2 := buildInOutIterator(obj, qs, clone, true)
or := iterator.NewOr()
or.AddSubIterator(it1)
or.AddSubIterator(it2)
it = or
case "out":
it = buildInOutIterator(obj, qs, subIt, false)
case "follow":
// Follow a morphism
arg, _ := obj.Get("_gremlin_values")
firstArg, _ := arg.Object().Get("0")
if isVertexChain(firstArg.Object()) {
return iterator.NewNull()
}
it = buildIteratorTreeHelper(firstArg.Object(), qs, subIt)
case "followr":
// Follow a morphism
arg, _ := obj.Get("_gremlin_followr")
if isVertexChain(arg.Object()) {
return iterator.NewNull()
}
it = buildIteratorTreeHelper(arg.Object(), qs, subIt)
case "in":
it = buildInOutIterator(obj, qs, subIt, true)
case "except":
arg, _ := obj.Get("_gremlin_values")
firstArg, _ := arg.Object().Get("0")
if !isVertexChain(firstArg.Object()) {
return iterator.NewNull()
}
allIt := qs.NodesAllIterator()
toComplementIt := buildIteratorTree(firstArg.Object(), qs)
notIt := iterator.NewNot(toComplementIt, allIt)
and := iterator.NewAnd(qs)
and.AddSubIterator(subIt)
and.AddSubIterator(notIt)
it = and
case "in_predicates":
it = buildInOutPredicateIterator(obj, qs, subIt, true)
case "out_predicates":
it = buildInOutPredicateIterator(obj, qs, subIt, false)
}
if it == nil {
panic("Iterator building does not catch the output iterator in some case.")
}
return it
}
