Exemple #1
0
func iterated(it graph.Iterator) []int {
	var res []int
	for graph.Next(it) {
		res = append(res, it.Result().(int))
	}
	return res
}
func (qs *QuadStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {
	switch it.Type() {
	case graph.LinksTo:
		return qs.optimizeLinksTo(it.(*iterator.LinksTo))

	}
	return it, false
}
Exemple #3
0
func iteratedQuads(qs graph.QuadStore, it graph.Iterator) []quad.Quad {
	var res ordered
	for graph.Next(it) {
		res = append(res, qs.Quad(it.Result()))
	}
	sort.Sort(res)
	return res
}
Exemple #4
0
func iteratedNames(qs graph.QuadStore, it graph.Iterator) []string {
	var res []string
	for graph.Next(it) {
		res = append(res, qs.NameOf(it.Result()))
	}
	sort.Strings(res)
	return res
}
Exemple #5
0
func OutputQueryShapeForIterator(it graph.Iterator, qs graph.QuadStore, outputMap map[string]interface{}) {
	s := &queryShape{
		qs:     qs,
		nodeID: 1,
	}

	node := s.MakeNode(it.Clone())
	s.AddNode(node)
	outputMap["nodes"] = s.nodes
	outputMap["links"] = s.links
}
// moveTagsTo() gets the tags for all of the src's subiterators and the
// src itself, and moves them to dst.
func moveTagsTo(dst graph.Iterator, src *And) {
	tags := src.getSubTags()
	for _, tag := range dst.Tagger().Tags() {
		if _, ok := tags[tag]; ok {
			delete(tags, tag)
		}
	}
	dt := dst.Tagger()
	for k := range tags {
		dt.Add(k)
	}
}
Exemple #7
0
func TestIterators(t *testing.T) {
	glog.Info("\n-----------\n")
	inst, opts, err := createInstance()
	defer inst.Close()

	if err != nil {
		t.Fatalf("failed to create instance: %v", err)
	}
	qs, _, _ := makeTestStore(simpleGraph, opts)
	if qs.Size() != 11 {
		t.Fatal("Incorrect number of quads")
	}

	var expected = []string{
		quad.Quad{"C", "follows", "B", ""}.String(),
		quad.Quad{"C", "follows", "D", ""}.String(),
	}

	it := qs.QuadIterator(quad.Subject, qs.ValueOf("C"))
	if got, ok := compareResults(qs, it, expected); !ok {
		t.Errorf("Unexpected iterated result, got:%v expect:%v", got, expected)
	}

	// Test contains
	it = qs.QuadIterator(quad.Label, qs.ValueOf("status_graph"))
	gqs := qs.(*QuadStore)
	key := gqs.createKeyForQuad(quad.Quad{"G", "status", "cool", "status_graph"})
	token := &Token{quadKind, key.StringID()}
	if !it.Contains(token) {
		t.Error("Contains failed")
	}

	// Test cloning an iterator
	var it2 graph.Iterator
	it2 = it.Clone()
	x := it2.Describe()
	y := it.Describe()

	if x.Name != y.Name {
		t.Errorf("Iterator Clone was not successful got: %v, expected: %v", x.Name, y.Name)
	}
}
Exemple #8
0
func iterateResults(qs graph.QuadStore, it graph.Iterator) []string {
	var res []string
	for graph.Next(it) {
		v := it.Result()
		if t, ok := v.(*Token); ok && t.Kind == nodeKind {
			res = append(res, qs.NameOf(it.Result()))
		} else {
			res = append(res, qs.Quad(it.Result()).String())
		}
	}
	sort.Strings(res)
	it.Reset()
	return res
}
Exemple #9
0
func buildInOutIterator(obj *otto.Object, qs graph.QuadStore, base graph.Iterator, isReverse bool) graph.Iterator {
	argList, _ := obj.Get("_gremlin_values")
	if argList.Class() != "GoArray" {
		glog.Errorln("How is arglist not an array? Return nothing.", argList.Class())
		return iterator.NewNull()
	}
	argArray := argList.Object()
	lengthVal, _ := argArray.Get("length")
	length, _ := lengthVal.ToInteger()
	var predicateNodeIterator graph.Iterator
	if length == 0 {
		predicateNodeIterator = qs.NodesAllIterator()
	} else {
		zero, _ := argArray.Get("0")
		predicateNodeIterator = buildIteratorFromValue(zero, qs)
	}
	if length >= 2 {
		var tags []string
		one, _ := argArray.Get("1")
		if one.IsString() {
			tags = append(tags, one.String())
		} else if one.Class() == "Array" {
			tags = stringsFrom(one.Object())
		}
		for _, tag := range tags {
			predicateNodeIterator.Tagger().Add(tag)
		}
	}

	in, out := quad.Subject, quad.Object
	if isReverse {
		in, out = out, in
	}
	lto := iterator.NewLinksTo(qs, base, in)
	and := iterator.NewAnd(qs)
	and.AddSubIterator(iterator.NewLinksTo(qs, predicateNodeIterator, quad.Predicate))
	and.AddSubIterator(lto)
	return iterator.NewHasA(qs, and, out)
}
Exemple #10
0
func (wk *worker) runIteratorToArray(it graph.Iterator, limit int) []map[string]string {
	output := make([]map[string]string, 0)
	n := 0
	it, _ = it.Optimize()
	for {
		select {
		case <-wk.kill:
			return nil
		default:
		}
		if !graph.Next(it) {
			break
		}
		tags := make(map[string]graph.Value)
		it.TagResults(tags)
		output = append(output, wk.tagsToValueMap(tags))
		n++
		if limit >= 0 && n >= limit {
			break
		}
		for it.NextPath() {
			select {
			case <-wk.kill:
				return nil
			default:
			}
			tags := make(map[string]graph.Value)
			it.TagResults(tags)
			output = append(output, wk.tagsToValueMap(tags))
			n++
			if limit >= 0 && n >= limit {
				break
			}
		}
	}
	it.Close()
	return output
}
Exemple #11
0
func (wk *worker) runIteratorToArrayNoTags(it graph.Iterator, limit int) []string {
	output := make([]string, 0)
	n := 0
	it, _ = it.Optimize()
	for {
		select {
		case <-wk.kill:
			return nil
		default:
		}
		if !graph.Next(it) {
			break
		}
		output = append(output, wk.qs.NameOf(it.Result()))
		n++
		if limit >= 0 && n >= limit {
			break
		}
	}
	it.Close()
	return output
}
// optimizeOrder(l) takes a list and returns a list, containing the same contents
// but with a new ordering, however it wishes.
func (it *And) optimizeOrder(its []graph.Iterator) []graph.Iterator {
	var (
		// bad contains iterators that can't be (efficiently) nexted, such as
		// graph.Optional or graph.Not. Separate them out and tack them on at the end.
		out, bad []graph.Iterator
		best     graph.Iterator
		bestCost = int64(1 << 62)
	)

	// Find the iterator with the projected "best" total cost.
	// Total cost is defined as The Next()ed iterator's cost to Next() out
	// all of it's contents, and to Contains() each of those against everyone
	// else.
	for _, root := range its {
		if _, canNext := root.(graph.Nexter); !canNext {
			bad = append(bad, root)
			continue
		}
		rootStats := root.Stats()
		cost := rootStats.NextCost
		for _, f := range its {
			if _, canNext := f.(graph.Nexter); !canNext {
				continue
			}
			if f == root {
				continue
			}
			stats := f.Stats()
			cost += stats.ContainsCost * (1 + (rootStats.Size / (stats.Size + 1)))
		}
		cost *= rootStats.Size
		if glog.V(3) {
			glog.V(3).Infoln("And:", it.UID(), "Root:", root.UID(), "Total Cost:", cost, "Best:", bestCost)
		}
		if cost < bestCost {
			best = root
			bestCost = cost
		}
	}
	if glog.V(3) {
		glog.V(3).Infoln("And:", it.UID(), "Choosing:", best.UID(), "Best:", bestCost)
	}

	// TODO(barakmich): Optimization of order need not stop here. Picking a smart
	// Contains() order based on probability of getting a false Contains() first is
	// useful (fail faster).

	// Put the best iterator (the one we wish to Next()) at the front...
	out = append(out, best)

	// ... push everyone else after...
	for _, it := range its {
		if _, canNext := it.(graph.Nexter); !canNext {
			continue
		}
		if it != best {
			out = append(out, it)
		}
	}

	// ...and finally, the difficult children on the end.
	return append(out, bad...)
}
Exemple #13
0
func TestIterator(t *testing.T) {
	tmpDir, err := ioutil.TempDir(os.TempDir(), "cayley_test")
	if err != nil {
		t.Fatalf("Could not create working directory: %v", err)
	}
	defer os.RemoveAll(tmpDir)
	t.Log(tmpDir)

	err = createNewLevelDB(tmpDir, nil)
	if err != nil {
		t.Fatal("Failed to create LevelDB database.")
	}

	qs, err := newQuadStore(tmpDir, nil)
	if qs == nil || err != nil {
		t.Error("Failed to create leveldb QuadStore.")
	}

	w, _ := writer.NewSingleReplication(qs, nil)
	w.AddQuadSet(makeQuadSet())
	var it graph.Iterator

	it = qs.NodesAllIterator()
	if it == nil {
		t.Fatal("Got nil iterator.")
	}

	size, exact := it.Size()
	if size <= 0 || size >= 20 {
		t.Errorf("Unexpected size, got:%d expect:(0, 20)", size)
	}
	if exact {
		t.Errorf("Got unexpected exact result.")
	}
	if typ := it.Type(); typ != graph.All {
		t.Errorf("Unexpected iterator type, got:%v expect:%v", typ, graph.All)
	}
	optIt, changed := it.Optimize()
	if changed || optIt != it {
		t.Errorf("Optimize unexpectedly changed iterator.")
	}

	expect := []string{
		"A",
		"B",
		"C",
		"D",
		"E",
		"F",
		"G",
		"follows",
		"status",
		"cool",
		"status_graph",
	}
	sort.Strings(expect)
	for i := 0; i < 2; i++ {
		got := iteratedNames(qs, it)
		sort.Strings(got)
		if !reflect.DeepEqual(got, expect) {
			t.Errorf("Unexpected iterated result on repeat %d, got:%v expect:%v", i, got, expect)
		}
		it.Reset()
	}

	for _, pq := range expect {
		if !it.Contains(qs.ValueOf(pq)) {
			t.Errorf("Failed to find and check %q correctly", pq)
		}
	}
	// FIXME(kortschak) Why does this fail?
	/*
		for _, pq := range []string{"baller"} {
			if it.Contains(qs.ValueOf(pq)) {
				t.Errorf("Failed to check %q correctly", pq)
			}
		}
	*/
	it.Reset()

	it = qs.QuadsAllIterator()
	graph.Next(it)
	q := qs.Quad(it.Result())
	set := makeQuadSet()
	var ok bool
	for _, t := range set {
		if t.String() == q.String() {
			ok = true
			break
		}
	}
	if !ok {
		t.Errorf("Failed to find %q during iteration, got:%q", q, set)
	}

	qs.Close()
}
Exemple #14
0
func (wk *worker) runIteratorWithCallback(it graph.Iterator, callback otto.Value, this otto.FunctionCall, limit int) {
	n := 0
	it, _ = it.Optimize()
	if glog.V(2) {
		b, err := json.MarshalIndent(it.Describe(), "", "  ")
		if err != nil {
			glog.V(2).Infof("failed to format description: %v", err)
		} else {
			glog.V(2).Infof("%s", b)
		}
	}
	for {
		select {
		case <-wk.kill:
			return
		default:
		}
		if !graph.Next(it) {
			break
		}
		tags := make(map[string]graph.Value)
		it.TagResults(tags)
		val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
		val, _ = callback.Call(this.This, val)
		n++
		if limit >= 0 && n >= limit {
			break
		}
		for it.NextPath() {
			select {
			case <-wk.kill:
				return
			default:
			}
			tags := make(map[string]graph.Value)
			it.TagResults(tags)
			val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
			val, _ = callback.Call(this.This, val)
			n++
			if limit >= 0 && n >= limit {
				break
			}
		}
	}
	it.Close()
}
Exemple #15
0
func (wk *worker) runIterator(it graph.Iterator) {
	if wk.wantShape() {
		iterator.OutputQueryShapeForIterator(it, wk.qs, wk.shape)
		return
	}
	it, _ = it.Optimize()
	if glog.V(2) {
		b, err := json.MarshalIndent(it.Describe(), "", "  ")
		if err != nil {
			glog.Infof("failed to format description: %v", err)
		} else {
			glog.Infof("%s", b)
		}
	}
	for {
		select {
		case <-wk.kill:
			return
		default:
		}
		if !graph.Next(it) {
			break
		}
		tags := make(map[string]graph.Value)
		it.TagResults(tags)
		if !wk.send(&Result{actualResults: tags}) {
			break
		}
		for it.NextPath() {
			select {
			case <-wk.kill:
				return
			default:
			}
			tags := make(map[string]graph.Value)
			it.TagResults(tags)
			if !wk.send(&Result{actualResults: tags}) {
				break
			}
		}
	}
	if glog.V(2) {
		bytes, _ := json.MarshalIndent(graph.DumpStats(it), "", "  ")
		glog.V(2).Infoln(string(bytes))
	}
	it.Close()
}
Exemple #16
0
func (s *queryShape) MakeNode(it graph.Iterator) *Node {
	n := Node{ID: s.nodeID}
	for _, tag := range it.Tagger().Tags() {
		n.Tags = append(n.Tags, tag)
	}
	for k := range it.Tagger().Fixed() {
		n.Tags = append(n.Tags, k)
	}

	switch it.Type() {
	case graph.And:
		for _, sub := range it.SubIterators() {
			s.nodeID++
			newNode := s.MakeNode(sub)
			if sub.Type() != graph.Or {
				s.StealNode(&n, newNode)
			} else {
				s.AddNode(newNode)
				s.AddLink(&Link{n.ID, newNode.ID, 0, 0})
			}
		}
	case graph.Fixed:
		n.IsFixed = true
		for graph.Next(it) {
			n.Values = append(n.Values, s.qs.NameOf(it.Result()))
		}
	case graph.HasA:
		hasa := it.(*HasA)
		s.PushHasa(n.ID, hasa.dir)
		s.nodeID++
		newNode := s.MakeNode(hasa.primaryIt)
		s.AddNode(newNode)
		s.RemoveHasa()
	case graph.Or:
		for _, sub := range it.SubIterators() {
			s.nodeID++
			newNode := s.MakeNode(sub)
			if sub.Type() == graph.Or {
				s.StealNode(&n, newNode)
			} else {
				s.AddNode(newNode)
				s.AddLink(&Link{n.ID, newNode.ID, 0, 0})
			}
		}
	case graph.LinksTo:
		n.IsLinkNode = true
		lto := it.(*LinksTo)
		s.nodeID++
		newNode := s.MakeNode(lto.primaryIt)
		hasaID, hasaDir := s.LastHasa()
		if (hasaDir == quad.Subject && lto.dir == quad.Object) ||
			(hasaDir == quad.Object && lto.dir == quad.Subject) {
			s.AddNode(newNode)
			if hasaDir == quad.Subject {
				s.AddLink(&Link{hasaID, newNode.ID, 0, n.ID})
			} else {
				s.AddLink(&Link{newNode.ID, hasaID, 0, n.ID})
			}
		} else if lto.primaryIt.Type() == graph.Fixed {
			s.StealNode(&n, newNode)
		} else {
			s.AddNode(newNode)
		}
	case graph.Optional:
		// Unsupported, for the moment
		fallthrough
	case graph.All:
	}
	return &n
}
Exemple #17
0
func printIterator(qs graph.QuadStore, it graph.Iterator) {
	for graph.Next(it) {
		glog.Infof("%v", qs.Quad(it.Result()))
	}
}
Exemple #18
0
func buildIteratorTreeHelper(obj *otto.Object, qs graph.QuadStore, base graph.Iterator) graph.Iterator {
	// TODO: Better error handling
	var (
		it    graph.Iterator
		subIt graph.Iterator
	)

	if prev, _ := obj.Get("_gremlin_prev"); !prev.IsObject() {
		subIt = base
	} else {
		subIt = buildIteratorTreeHelper(prev.Object(), qs, base)
	}

	stringArgs := propertiesOf(obj, "string_args")
	val, _ := obj.Get("_gremlin_type")
	switch val.String() {
	case "vertex":
		if len(stringArgs) == 0 {
			it = qs.NodesAllIterator()
		} else {
			fixed := qs.FixedIterator()
			for _, name := range stringArgs {
				fixed.Add(qs.ValueOf(name))
			}
			it = fixed
		}
	case "tag":
		it = subIt
		for _, tag := range stringArgs {
			it.Tagger().Add(tag)
		}
	case "save":
		all := qs.NodesAllIterator()
		if len(stringArgs) > 2 || len(stringArgs) == 0 {
			return iterator.NewNull()
		}
		if len(stringArgs) == 2 {
			all.Tagger().Add(stringArgs[1])
		} else {
			all.Tagger().Add(stringArgs[0])
		}
		predFixed := qs.FixedIterator()
		predFixed.Add(qs.ValueOf(stringArgs[0]))
		subAnd := iterator.NewAnd(qs)
		subAnd.AddSubIterator(iterator.NewLinksTo(qs, predFixed, quad.Predicate))
		subAnd.AddSubIterator(iterator.NewLinksTo(qs, all, quad.Object))
		hasa := iterator.NewHasA(qs, subAnd, quad.Subject)
		and := iterator.NewAnd(qs)
		and.AddSubIterator(hasa)
		and.AddSubIterator(subIt)
		it = and
	case "saver":
		all := qs.NodesAllIterator()
		if len(stringArgs) > 2 || len(stringArgs) == 0 {
			return iterator.NewNull()
		}
		if len(stringArgs) == 2 {
			all.Tagger().Add(stringArgs[1])
		} else {
			all.Tagger().Add(stringArgs[0])
		}
		predFixed := qs.FixedIterator()
		predFixed.Add(qs.ValueOf(stringArgs[0]))
		subAnd := iterator.NewAnd(qs)
		subAnd.AddSubIterator(iterator.NewLinksTo(qs, predFixed, quad.Predicate))
		subAnd.AddSubIterator(iterator.NewLinksTo(qs, all, quad.Subject))
		hasa := iterator.NewHasA(qs, subAnd, quad.Object)
		and := iterator.NewAnd(qs)
		and.AddSubIterator(hasa)
		and.AddSubIterator(subIt)
		it = and
	case "has":
		fixed := qs.FixedIterator()
		if len(stringArgs) < 2 {
			return iterator.NewNull()
		}
		for _, name := range stringArgs[1:] {
			fixed.Add(qs.ValueOf(name))
		}
		predFixed := qs.FixedIterator()
		predFixed.Add(qs.ValueOf(stringArgs[0]))
		subAnd := iterator.NewAnd(qs)
		subAnd.AddSubIterator(iterator.NewLinksTo(qs, predFixed, quad.Predicate))
		subAnd.AddSubIterator(iterator.NewLinksTo(qs, fixed, quad.Object))
		hasa := iterator.NewHasA(qs, subAnd, quad.Subject)
		and := iterator.NewAnd(qs)
		and.AddSubIterator(hasa)
		and.AddSubIterator(subIt)
		it = and
	case "morphism":
		it = base
	case "and":
		arg, _ := obj.Get("_gremlin_values")
		firstArg, _ := arg.Object().Get("0")
		if !isVertexChain(firstArg.Object()) {
			return iterator.NewNull()
		}
		argIt := buildIteratorTree(firstArg.Object(), qs)

		and := iterator.NewAnd(qs)
		and.AddSubIterator(subIt)
		and.AddSubIterator(argIt)
		it = and
	case "back":
		arg, _ := obj.Get("_gremlin_back_chain")
		argIt := buildIteratorTree(arg.Object(), qs)
		and := iterator.NewAnd(qs)
		and.AddSubIterator(subIt)
		and.AddSubIterator(argIt)
		it = and
	case "is":
		fixed := qs.FixedIterator()
		for _, name := range stringArgs {
			fixed.Add(qs.ValueOf(name))
		}
		and := iterator.NewAnd(qs)
		and.AddSubIterator(fixed)
		and.AddSubIterator(subIt)
		it = and
	case "or":
		arg, _ := obj.Get("_gremlin_values")
		firstArg, _ := arg.Object().Get("0")
		if !isVertexChain(firstArg.Object()) {
			return iterator.NewNull()
		}
		argIt := buildIteratorTree(firstArg.Object(), qs)

		or := iterator.NewOr()
		or.AddSubIterator(subIt)
		or.AddSubIterator(argIt)
		it = or
	case "both":
		// Hardly the most efficient pattern, but the most general.
		// Worth looking into an Optimize() optimization here.
		clone := subIt.Clone()
		it1 := buildInOutIterator(obj, qs, subIt, false)
		it2 := buildInOutIterator(obj, qs, clone, true)

		or := iterator.NewOr()
		or.AddSubIterator(it1)
		or.AddSubIterator(it2)
		it = or
	case "out":
		it = buildInOutIterator(obj, qs, subIt, false)
	case "follow":
		// Follow a morphism
		arg, _ := obj.Get("_gremlin_values")
		firstArg, _ := arg.Object().Get("0")
		if isVertexChain(firstArg.Object()) {
			return iterator.NewNull()
		}
		it = buildIteratorTreeHelper(firstArg.Object(), qs, subIt)
	case "followr":
		// Follow a morphism
		arg, _ := obj.Get("_gremlin_followr")
		if isVertexChain(arg.Object()) {
			return iterator.NewNull()
		}
		it = buildIteratorTreeHelper(arg.Object(), qs, subIt)
	case "in":
		it = buildInOutIterator(obj, qs, subIt, true)
	case "except":
		arg, _ := obj.Get("_gremlin_values")
		firstArg, _ := arg.Object().Get("0")
		if !isVertexChain(firstArg.Object()) {
			return iterator.NewNull()
		}

		allIt := qs.NodesAllIterator()
		toComplementIt := buildIteratorTree(firstArg.Object(), qs)
		notIt := iterator.NewNot(toComplementIt, allIt)

		and := iterator.NewAnd(qs)
		and.AddSubIterator(subIt)
		and.AddSubIterator(notIt)
		it = and
	case "in_predicates":
		it = buildInOutPredicateIterator(obj, qs, subIt, true)
	case "out_predicates":
		it = buildInOutPredicateIterator(obj, qs, subIt, false)
	}
	if it == nil {
		panic("Iterator building does not catch the output iterator in some case.")
	}
	return it
}