func (ts *LevelDBTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {
switch it.Type() {
case "linksto":
return ts.optimizeLinksTo(it.(*graph.LinksToIterator))
}
return it, false
}
func (m *MongoIterator) Clone() graph.Iterator {
var newM graph.Iterator
if m.isAll {
newM = NewMongoAllIterator(m.ts, m.collection)
} else {
newM = NewMongoIterator(m.ts, m.collection, m.dir, m.hash)
}
newM.CopyTagsFrom(m)
return newM
}
func extractValuesFromIterator(ts graph.TripleStore, it graph.Iterator) []string {
var output []string
for {
val, ok := it.Next()
if !ok {
break
}
output = append(output, ts.GetNameFor(val))
}
return output
}
func extractTripleFromIterator(ts graph.TripleStore, it graph.Iterator) []string {
var output []string
for {
val, ok := it.Next()
if !ok {
break
}
output = append(output, ts.GetTriple(val).ToString())
}
return output
}
func TestOptimize(t *testing.T) {
var ts *LevelDBTripleStore
var lto graph.Iterator
var tmpDir string
Convey("Given a prepared database", t, func() {
tmpDir, _ = ioutil.TempDir(os.TempDir(), "cayley_test")
t.Log(tmpDir)
defer os.RemoveAll(tmpDir)
ok := CreateNewLevelDB(tmpDir)
So(ok, ShouldBeTrue)
ts = NewDefaultLevelDBTripleStore(tmpDir, nil)
ts.AddTripleSet(makeTripleSet())
Convey("With an linksto-fixed pair", func() {
fixed := ts.MakeFixed()
fixed.AddValue(ts.GetIdFor("F"))
fixed.AddTag("internal")
lto = graph.NewLinksToIterator(ts, fixed, "o")
Convey("Creates an appropriate iterator", func() {
oldIt := lto.Clone()
newIt, ok := lto.Optimize()
So(ok, ShouldBeTrue)
So(newIt.Type(), ShouldEqual, "leveldb")
Convey("Containing the right things", func() {
afterOp := extractTripleFromIterator(ts, newIt)
beforeOp := extractTripleFromIterator(ts, oldIt)
sort.Strings(afterOp)
sort.Strings(beforeOp)
So(afterOp, ShouldResemble, beforeOp)
})
Convey("With the correct tags", func() {
oldIt.Next()
newIt.Next()
oldResults := make(map[string]graph.TSVal)
oldIt.TagResults(&oldResults)
newResults := make(map[string]graph.TSVal)
oldIt.TagResults(&newResults)
So(newResults, ShouldResemble, oldResults)
})
})
})
})
}
func (m *MqlQuery) buildIteratorTreeInternal(query interface{}, path MqlPath) (graph.Iterator, error) {
var it graph.Iterator
var err error
err = nil
switch t := query.(type) {
case bool:
// for JSON booleans
// Treat the bool as a string and call it a day.
// Things which are really bool-like are special cases and will be dealt with separately.
if t {
it = m.buildFixed("true")
}
it = m.buildFixed("false")
case float64:
// for JSON numbers
// Damn you, Javascript, and your lack of integer values.
if math.Floor(t) == t {
// Treat it like an integer.
it = m.buildFixed(fmt.Sprintf("%d", t))
} else {
it = m.buildFixed(fmt.Sprintf("%f", t))
}
case string:
// for JSON strings
it = m.buildFixed(t)
case []interface{}:
// for JSON arrays
m.isRepeated[path] = true
if len(t) == 0 {
it = m.buildResultIterator(path)
} else if len(t) == 1 {
it, err = m.buildIteratorTreeInternal(t[0], path)
} else {
err = errors.New(fmt.Sprintf("Multiple fields at location root%s", path.DisplayString()))
}
case map[string]interface{}:
// for JSON objects
it, err = m.buildIteratorTreeMapInternal(t, path)
case nil:
it = m.buildResultIterator(path)
default:
log.Fatal("Unknown JSON type?", query)
}
if err != nil {
return nil, err
}
it.AddTag(string(path))
return it, nil
}
func buildInOutIterator(obj *otto.Object, ts graph.TripleStore, base graph.Iterator, isReverse bool) graph.Iterator {
argList, _ := obj.Get("_gremlin_values")
if argList.Class() != "GoArray" {
glog.Errorln("How is arglist not an array? Return nothing.", argList.Class())
return graph.NewNullIterator()
}
argArray := argList.Object()
lengthVal, _ := argArray.Get("length")
length, _ := lengthVal.ToInteger()
var predicateNodeIterator graph.Iterator
if length == 0 {
predicateNodeIterator = ts.GetNodesAllIterator()
} else {
zero, _ := argArray.Get("0")
predicateNodeIterator = buildIteratorFromValue(zero, ts)
}
if length >= 2 {
var tags []string
one, _ := argArray.Get("1")
if one.IsString() {
s, _ := one.ToString()
tags = append(tags, s)
} else if one.Class() == "Array" {
tags = makeListOfStringsFromArrayValue(one.Object())
}
for _, tag := range tags {
predicateNodeIterator.AddTag(tag)
}
}
in, out := "s", "o"
if isReverse {
in, out = out, in
}
lto := graph.NewLinksToIterator(ts, base, in)
and := graph.NewAndIterator()
and.AddSubIterator(graph.NewLinksToIterator(ts, predicateNodeIterator, "p"))
and.AddSubIterator(lto)
return graph.NewHasaIterator(ts, and, out)
}
func runIteratorToArrayNoTags(it graph.Iterator, ses *GremlinSession, limit int) []string {
output := make([]string, 0)
count := 0
it, _ = it.Optimize()
for {
if ses.doHalt {
return nil
}
val, ok := it.Next()
if !ok {
break
}
output = append(output, ses.ts.GetNameFor(val))
count++
if limit >= 0 && count >= limit {
break
}
}
it.Close()
return output
}
func runIteratorOnSession(it graph.Iterator, ses *GremlinSession) {
if ses.lookingForQueryShape {
graph.OutputQueryShapeForIterator(it, ses.ts, &(ses.queryShape))
return
}
it, _ = it.Optimize()
glog.V(2).Infoln(it.DebugString(0))
for {
// TODO(barakmich): Better halting.
if ses.doHalt {
return
}
_, ok := it.Next()
if !ok {
break
}
tags := make(map[string]graph.TSVal)
it.TagResults(&tags)
cont := ses.SendResult(&GremlinResult{metaresult: false, err: "", val: nil, actualResults: &tags})
if !cont {
break
}
for it.NextResult() == true {
if ses.doHalt {
return
}
tags := make(map[string]graph.TSVal)
it.TagResults(&tags)
cont := ses.SendResult(&GremlinResult{metaresult: false, err: "", val: nil, actualResults: &tags})
if !cont {
break
}
}
}
it.Close()
}
func runIteratorWithCallback(it graph.Iterator, ses *GremlinSession, callback otto.Value, this otto.FunctionCall, limit int) {
count := 0
it, _ = it.Optimize()
for {
if ses.doHalt {
return
}
_, ok := it.Next()
if !ok {
break
}
tags := make(map[string]graph.TSVal)
it.TagResults(&tags)
val, _ := this.Otto.ToValue(tagsToValueMap(tags, ses))
val, _ = callback.Call(this.This, val)
count++
if limit >= 0 && count >= limit {
break
}
for it.NextResult() == true {
if ses.doHalt {
return
}
tags := make(map[string]graph.TSVal)
it.TagResults(&tags)
val, _ := this.Otto.ToValue(tagsToValueMap(tags, ses))
val, _ = callback.Call(this.This, val)
count++
if limit >= 0 && count >= limit {
break
}
}
}
it.Close()
}
func runIteratorToArray(it graph.Iterator, ses *GremlinSession, limit int) []map[string]string {
output := make([]map[string]string, 0)
count := 0
it, _ = it.Optimize()
for {
if ses.doHalt {
return nil
}
_, ok := it.Next()
if !ok {
break
}
tags := make(map[string]graph.TSVal)
it.TagResults(&tags)
output = append(output, tagsToValueMap(tags, ses))
count++
if limit >= 0 && count >= limit {
break
}
for it.NextResult() == true {
if ses.doHalt {
return nil
}
tags := make(map[string]graph.TSVal)
it.TagResults(&tags)
output = append(output, tagsToValueMap(tags, ses))
count++
if limit >= 0 && count >= limit {
break
}
}
}
it.Close()
return output
}
func TestSetIterator(t *testing.T) {
var ts *LevelDBTripleStore
var tmpDir string
Convey("Given a prepared database", t, func() {
tmpDir, _ = ioutil.TempDir(os.TempDir(), "cayley_test")
t.Log(tmpDir)
defer os.RemoveAll(tmpDir)
ok := CreateNewLevelDB(tmpDir)
So(ok, ShouldBeTrue)
ts = NewDefaultLevelDBTripleStore(tmpDir, nil)
ts.AddTripleSet(makeTripleSet())
var it graph.Iterator
Convey("Can create a subject iterator", func() {
it = ts.GetTripleIterator("s", ts.GetIdFor("C"))
Convey("Containing the right things", func() {
expected := []string{
graph.MakeTriple("C", "follows", "B", "").ToString(),
graph.MakeTriple("C", "follows", "D", "").ToString(),
}
actual := extractTripleFromIterator(ts, it)
sort.Strings(actual)
sort.Strings(expected)
So(actual, ShouldResemble, expected)
})
Convey("And checkable", func() {
and := graph.NewAndIterator()
and.AddSubIterator(ts.GetTriplesAllIterator())
and.AddSubIterator(it)
expected := []string{
graph.MakeTriple("C", "follows", "B", "").ToString(),
graph.MakeTriple("C", "follows", "D", "").ToString(),
}
actual := extractTripleFromIterator(ts, and)
sort.Strings(actual)
sort.Strings(expected)
So(actual, ShouldResemble, expected)
})
Reset(func() {
it.Reset()
})
})
Convey("Can create an object iterator", func() {
it = ts.GetTripleIterator("o", ts.GetIdFor("F"))
Convey("Containing the right things", func() {
expected := []string{
graph.MakeTriple("B", "follows", "F", "").ToString(),
graph.MakeTriple("E", "follows", "F", "").ToString(),
}
actual := extractTripleFromIterator(ts, it)
sort.Strings(actual)
sort.Strings(expected)
So(actual, ShouldResemble, expected)
})
Convey("Mutually and-checkable", func() {
and := graph.NewAndIterator()
and.AddSubIterator(ts.GetTripleIterator("s", ts.GetIdFor("B")))
and.AddSubIterator(it)
expected := []string{
graph.MakeTriple("B", "follows", "F", "").ToString(),
}
actual := extractTripleFromIterator(ts, and)
sort.Strings(actual)
sort.Strings(expected)
So(actual, ShouldResemble, expected)
})
})
Convey("Can create a predicate iterator", func() {
it = ts.GetTripleIterator("p", ts.GetIdFor("status"))
Convey("Containing the right things", func() {
expected := []string{
graph.MakeTriple("B", "status", "cool", "status_graph").ToString(),
graph.MakeTriple("D", "status", "cool", "status_graph").ToString(),
graph.MakeTriple("G", "status", "cool", "status_graph").ToString(),
}
actual := extractTripleFromIterator(ts, it)
sort.Strings(actual)
sort.Strings(expected)
So(actual, ShouldResemble, expected)
})
})
Convey("Can create a provenance iterator", func() {
it = ts.GetTripleIterator("c", ts.GetIdFor("status_graph"))
Convey("Containing the right things", func() {
expected := []string{
graph.MakeTriple("B", "status", "cool", "status_graph").ToString(),
graph.MakeTriple("D", "status", "cool", "status_graph").ToString(),
graph.MakeTriple("G", "status", "cool", "status_graph").ToString(),
}
actual := extractTripleFromIterator(ts, it)
sort.Strings(actual)
sort.Strings(expected)
So(actual, ShouldResemble, expected)
})
Convey("Can be cross-checked", func() {
and := graph.NewAndIterator()
// Order is important
and.AddSubIterator(ts.GetTripleIterator("s", ts.GetIdFor("B")))
and.AddSubIterator(it)
expected := []string{
graph.MakeTriple("B", "status", "cool", "status_graph").ToString(),
}
actual := extractTripleFromIterator(ts, and)
So(actual, ShouldResemble, expected)
})
Convey("Can check against other iterators", func() {
and := graph.NewAndIterator()
// Order is important
and.AddSubIterator(it)
and.AddSubIterator(ts.GetTripleIterator("s", ts.GetIdFor("B")))
expected := []string{
graph.MakeTriple("B", "status", "cool", "status_graph").ToString(),
}
actual := extractTripleFromIterator(ts, and)
So(actual, ShouldResemble, expected)
})
Reset(func() {
it.Reset()
})
})
Reset(func() {
ts.Close()
})
})
}
func TestAllIterator(t *testing.T) {
var ts *LevelDBTripleStore
Convey("Given a prepared database", t, func() {
tmpDir, _ := ioutil.TempDir(os.TempDir(), "cayley_test")
t.Log(tmpDir)
defer os.RemoveAll(tmpDir)
ok := CreateNewLevelDB(tmpDir)
So(ok, ShouldBeTrue)
ts = NewDefaultLevelDBTripleStore(tmpDir, nil)
ts.AddTripleSet(makeTripleSet())
var it graph.Iterator
Convey("Can create an all iterator for nodes", func() {
it = ts.GetNodesAllIterator()
So(it, ShouldNotBeNil)
Convey("Has basics", func() {
size, accurate := it.Size()
So(size, ShouldBeBetween, 0, 20)
So(accurate, ShouldBeFalse)
So(it.Type(), ShouldEqual, "all")
re_it, ok := it.Optimize()
So(ok, ShouldBeFalse)
So(re_it, ShouldPointTo, it)
})
Convey("Iterates all nodes", func() {
expected := []string{
"A",
"B",
"C",
"D",
"E",
"F",
"G",
"follows",
"status",
"cool",
"status_graph",
}
sort.Strings(expected)
actual := extractValuesFromIterator(ts, it)
sort.Strings(actual)
So(actual, ShouldResemble, expected)
it.Reset()
actual = extractValuesFromIterator(ts, it)
sort.Strings(actual)
So(actual, ShouldResemble, expected)
})
Convey("Contains a couple nodes", func() {
So(it.Check(ts.GetIdFor("A")), ShouldBeTrue)
So(it.Check(ts.GetIdFor("cool")), ShouldBeTrue)
//So(it.Check(ts.GetIdFor("baller")), ShouldBeFalse)
})
Reset(func() {
it.Reset()
})
})
Convey("Can create an all iterator for edges", func() {
it := ts.GetTriplesAllIterator()
So(it, ShouldNotBeNil)
Convey("Has basics", func() {
size, accurate := it.Size()
So(size, ShouldBeBetween, 0, 20)
So(accurate, ShouldBeFalse)
So(it.Type(), ShouldEqual, "all")
re_it, ok := it.Optimize()
So(ok, ShouldBeFalse)
So(re_it, ShouldPointTo, it)
})
Convey("Iterates an edge", func() {
edge_val, _ := it.Next()
triple := ts.GetTriple(edge_val)
set := makeTripleSet()
var string_set []string
for _, t := range set {
string_set = append(string_set, t.ToString())
}
So(triple.ToString(), ShouldBeIn, string_set)
})
Reset(func() {
ts.Close()
})
})
})
}
func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph.Iterator) graph.Iterator {
var it graph.Iterator
it = base
// TODO: Better error handling
kindVal, _ := obj.Get("_gremlin_type")
stringArgs := getStringArgs(obj)
var subIt graph.Iterator
prevVal, _ := obj.Get("_gremlin_prev")
if !prevVal.IsObject() {
subIt = base
} else {
subIt = buildIteratorTreeHelper(prevVal.Object(), ts, base)
}
kind, _ := kindVal.ToString()
switch kind {
case "vertex":
if len(stringArgs) == 0 {
it = ts.GetNodesAllIterator()
} else {
fixed := ts.MakeFixed()
for _, name := range stringArgs {
fixed.AddValue(ts.GetIdFor(name))
}
it = fixed
}
case "tag":
it = subIt
for _, tag := range stringArgs {
it.AddTag(tag)
}
case "save":
all := ts.GetNodesAllIterator()
if len(stringArgs) > 2 || len(stringArgs) == 0 {
return graph.NewNullIterator()
}
if len(stringArgs) == 2 {
all.AddTag(stringArgs[1])
} else {
all.AddTag(stringArgs[0])
}
predFixed := ts.MakeFixed()
predFixed.AddValue(ts.GetIdFor(stringArgs[0]))
subAnd := graph.NewAndIterator()
subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p"))
subAnd.AddSubIterator(graph.NewLinksToIterator(ts, all, "o"))
hasa := graph.NewHasaIterator(ts, subAnd, "s")
and := graph.NewAndIterator()
and.AddSubIterator(hasa)
and.AddSubIterator(subIt)
it = and
case "saver":
all := ts.GetNodesAllIterator()
if len(stringArgs) > 2 || len(stringArgs) == 0 {
return graph.NewNullIterator()
}
if len(stringArgs) == 2 {
all.AddTag(stringArgs[1])
} else {
all.AddTag(stringArgs[0])
}
predFixed := ts.MakeFixed()
predFixed.AddValue(ts.GetIdFor(stringArgs[0]))
subAnd := graph.NewAndIterator()
subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p"))
subAnd.AddSubIterator(graph.NewLinksToIterator(ts, all, "s"))
hasa := graph.NewHasaIterator(ts, subAnd, "o")
and := graph.NewAndIterator()
and.AddSubIterator(hasa)
and.AddSubIterator(subIt)
it = and
case "has":
fixed := ts.MakeFixed()
if len(stringArgs) < 2 {
return graph.NewNullIterator()
}
for _, name := range stringArgs[1:] {
fixed.AddValue(ts.GetIdFor(name))
}
predFixed := ts.MakeFixed()
predFixed.AddValue(ts.GetIdFor(stringArgs[0]))
subAnd := graph.NewAndIterator()
subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p"))
subAnd.AddSubIterator(graph.NewLinksToIterator(ts, fixed, "o"))
hasa := graph.NewHasaIterator(ts, subAnd, "s")
and := graph.NewAndIterator()
and.AddSubIterator(hasa)
and.AddSubIterator(subIt)
it = and
case "morphism":
it = base
case "and":
arg, _ := obj.Get("_gremlin_values")
firstArg, _ := arg.Object().Get("0")
if !isVertexChain(firstArg.Object()) {
return graph.NewNullIterator()
}
argIt := buildIteratorTree(firstArg.Object(), ts)
and := graph.NewAndIterator()
and.AddSubIterator(subIt)
and.AddSubIterator(argIt)
it = and
case "back":
arg, _ := obj.Get("_gremlin_back_chain")
argIt := buildIteratorTree(arg.Object(), ts)
and := graph.NewAndIterator()
and.AddSubIterator(subIt)
and.AddSubIterator(argIt)
it = and
case "is":
fixed := ts.MakeFixed()
for _, name := range stringArgs {
fixed.AddValue(ts.GetIdFor(name))
}
and := graph.NewAndIterator()
and.AddSubIterator(fixed)
and.AddSubIterator(subIt)
it = and
case "or":
arg, _ := obj.Get("_gremlin_values")
firstArg, _ := arg.Object().Get("0")
if !isVertexChain(firstArg.Object()) {
return graph.NewNullIterator()
}
argIt := buildIteratorTree(firstArg.Object(), ts)
or := graph.NewOrIterator()
or.AddSubIterator(subIt)
or.AddSubIterator(argIt)
it = or
case "both":
// Hardly the most efficient pattern, but the most general.
// Worth looking into an Optimize() optimization here.
clone := subIt.Clone()
it1 := buildInOutIterator(obj, ts, subIt, false)
it2 := buildInOutIterator(obj, ts, clone, true)
or := graph.NewOrIterator()
or.AddSubIterator(it1)
or.AddSubIterator(it2)
it = or
case "out":
it = buildInOutIterator(obj, ts, subIt, false)
case "follow":
// Follow a morphism
arg, _ := obj.Get("_gremlin_values")
firstArg, _ := arg.Object().Get("0")
if isVertexChain(firstArg.Object()) {
return graph.NewNullIterator()
}
it = buildIteratorTreeHelper(firstArg.Object(), ts, subIt)
case "followr":
// Follow a morphism
arg, _ := obj.Get("_gremlin_followr")
if isVertexChain(arg.Object()) {
return graph.NewNullIterator()
}
it = buildIteratorTreeHelper(arg.Object(), ts, subIt)
case "in":
it = buildInOutIterator(obj, ts, subIt, true)
}
return it
}