Beispiel #1
0
func (qs *QuadStore) indexOf(t quad.Quad) (int64, bool) {
	min := maxInt
	var tree *b.Tree
	for d := quad.Subject; d <= quad.Label; d++ {
		sid := t.Get(d)
		if d == quad.Label && sid == nil {
			continue
		}
		id, ok := qs.idMap[quad.StringOf(sid)]
		// If we've never heard about a node, it must not exist
		if !ok {
			return 0, false
		}
		index, ok := qs.index.Get(d, id)
		if !ok {
			// If it's never been indexed in this direction, it can't exist.
			return 0, false
		}
		if l := index.Len(); l < min {
			min, tree = l, index
		}
	}

	it := NewIterator(tree, qs, 0, nil)
	for it.Next() {
		if t == qs.log[it.result].Quad {
			return it.result, true
		}
	}
	return 0, false
}
Beispiel #2
0
func (qs *QuadStore) AddDelta(d graph.Delta) error {
	if _, exists := qs.indexOf(d.Quad); exists {
		return &graph.DeltaError{Delta: d, Err: graph.ErrQuadExists}
	}
	qid := qs.nextQuadID
	qs.log = append(qs.log, LogEntry{
		ID:        d.ID.Int(),
		Quad:      d.Quad,
		Action:    d.Action,
		Timestamp: d.Timestamp})
	qs.size++
	qs.nextQuadID++

	for dir := quad.Subject; dir <= quad.Label; dir++ {
		sid := d.Quad.Get(dir)
		if dir == quad.Label && sid == nil {
			continue
		}
		ssid := quad.StringOf(sid)
		if _, ok := qs.idMap[ssid]; !ok {
			qs.idMap[ssid] = qs.nextID
			qs.revIDMap[qs.nextID] = sid
			qs.nextID++
		}
		id := qs.idMap[ssid]
		tree := qs.index.Tree(dir, id)
		tree.Set(qid, struct{}{})
	}

	// TODO(barakmich): Add VIP indexing
	return nil
}
Beispiel #3
0
func quadValueToNative(v quad.Value) interface{} {
	out := v.Native()
	if nv, ok := out.(quad.Value); ok && v == nv {
		return quad.StringOf(v)
	}
	return out
}
Beispiel #4
0
// Here's the non-boilerplate part of the ValueComparison iterator. Given a value
// and our operator, determine whether or not we meet the requirement.
func (it *Comparison) doComparison(val graph.Value) bool {
	qval := it.qs.NameOf(val)
	switch cVal := it.val.(type) {
	case quad.Raw:
		return RunStrOp(quad.StringOf(qval), it.op, string(cVal))
	case quad.Int:
		if cVal2, ok := qval.(quad.Int); ok {
			return RunIntOp(cVal2, it.op, cVal)
		}
		return false
	case quad.Float:
		if cVal2, ok := qval.(quad.Float); ok {
			return RunFloatOp(cVal2, it.op, cVal)
		}
		return false
	case quad.String:
		if cVal2, ok := qval.(quad.String); ok {
			return RunStrOp(string(cVal2), it.op, string(cVal))
		}
		return false
	case quad.BNode:
		if cVal2, ok := qval.(quad.BNode); ok {
			return RunStrOp(string(cVal2), it.op, string(cVal))
		}
		return false
	case quad.IRI:
		if cVal2, ok := qval.(quad.IRI); ok {
			return RunStrOp(string(cVal2), it.op, string(cVal))
		}
		return false
	case quad.Time:
		if cVal2, ok := qval.(quad.Time); ok {
			return RunTimeOp(time.Time(cVal2), it.op, time.Time(cVal))
		}
		return false
	default:
		return RunStrOp(quad.StringOf(qval), it.op, quad.StringOf(it.val))
	}
}
Beispiel #5
0
func TestLoadOneQuad(t testing.TB, gen DatabaseFunc) {
	qs, opts, closer := gen(t)
	defer closer()

	w := MakeWriter(t, qs, opts)

	err := w.AddQuad(quad.MakeRaw(
		"Something",
		"points_to",
		"Something Else",
		"context",
	))
	require.Nil(t, err)
	for _, pq := range []string{"Something", "points_to", "Something Else", "context"} {
		got := quad.StringOf(qs.NameOf(qs.ValueOf(quad.Raw(pq))))
		require.Equal(t, pq, got, "Failed to roundtrip %q", pq)
	}
	require.Equal(t, int64(1), qs.Size(), "Unexpected quadstore size")
}
Beispiel #6
0
func NewIterator(qs *QuadStore, k string, d quad.Direction, val graph.Value) *Iterator {
	t := val.(*Token)
	if t == nil {
		clog.Errorf("Token == nil")
	}
	if t.Kind != nodeKind {
		clog.Errorf("Cannot create an iterator from a non-node value")
		return &Iterator{done: true}
	}
	if k != nodeKind && k != quadKind {
		clog.Errorf("Cannot create iterator for unknown kind")
		return &Iterator{done: true}
	}
	if qs.context == nil {
		clog.Errorf("Cannot create iterator without a valid context")
		return &Iterator{done: true}
	}
	name := quad.StringOf(qs.NameOf(t))

	// The number of references to this node is held in the nodes entity
	key := qs.createKeyFromToken(t)
	foundNode := new(NodeEntry)
	err := datastore.Get(qs.context, key, foundNode)
	if err != nil && err != datastore.ErrNoSuchEntity {
		clog.Errorf("Error: %v", err)
		return &Iterator{done: true}
	}
	size := foundNode.Size

	return &Iterator{
		uid:   iterator.NextUID(),
		name:  name,
		dir:   d,
		qs:    qs,
		size:  size,
		isAll: false,
		kind:  k,
		hash:  t.Hash,
		done:  false,
	}
}
Beispiel #7
0
func toStrings(objs []interface{}) []string {
	if len(objs) == 0 {
		return nil
	}
	var out = make([]string, 0, len(objs))
	for _, o := range objs {
		switch v := o.(type) {
		case string:
			out = append(out, v)
		case quad.Value:
			out = append(out, quad.StringOf(v))
		case []string:
			out = append(out, v...)
		case []interface{}:
			out = append(out, toStrings(v)...)
		default:
			panic(fmt.Errorf("expected string, got: %T", o))
		}
	}
	return out
}
Beispiel #8
0
func (qs *QuadStore) ValueOf(name quad.Value) graph.Value {
	return iterator.Int64Node(qs.idMap[quad.StringOf(name)])
}
Beispiel #9
0
func (q *Query) treeifyResult(tags map[string]graph.Value) map[ResultPath]string {
	// Transform the map into something a little more interesting.
	results := make(map[Path]string)
	for k, v := range tags {
		if v == nil {
			continue
		}
		results[Path(k)] = quad.StringOf(q.ses.qs.NameOf(v))
	}
	resultPaths := make(map[ResultPath]string)
	for k, v := range results {
		resultPaths[k.ToResultPathFromMap(results)] = v
	}

	paths := make([]ResultPath, 0, len(resultPaths))
	for path := range resultPaths {
		paths = append(paths, path)
	}
	sort.Sort(byRecordLength(paths))

	// Build Structure
	for _, path := range paths {
		currentPath := path.getPath()
		value := resultPaths[path]
		namePath := path.AppendValue(value)
		if _, ok := q.queryResult[namePath]; !ok {
			targetPath, key := path.splitLastPath()
			if path == "" {
				targetPath, key = "", value
				if _, ok := q.queryResult[""][value]; !ok {
					q.resultOrder = append(q.resultOrder, value)
				}
			}
			if _, ok := q.queryStructure[currentPath]; ok {
				// If there's substructure, then copy that in.
				newStruct := q.copyPathStructure(currentPath)
				if q.isRepeated[currentPath] && currentPath != "" {
					switch t := q.queryResult[targetPath][key].(type) {
					case nil:
						x := make([]interface{}, 0)
						x = append(x, newStruct)
						q.queryResult[targetPath][key] = x
						q.queryResult[namePath] = newStruct
					case []interface{}:
						q.queryResult[targetPath][key] = append(t, newStruct)
						q.queryResult[namePath] = newStruct
					}

				} else {
					q.queryResult[namePath] = newStruct
					q.queryResult[targetPath][key] = newStruct
				}
			}
		}
	}

	// Fill values
	for _, path := range paths {
		currentPath := path.getPath()
		value, ok := resultPaths[path]
		if !ok {
			continue
		}
		namePath := path.AppendValue(value)
		if _, ok := q.queryStructure[currentPath]; ok {
			// We're dealing with ids.
			if _, ok := q.queryResult[namePath]["id"]; ok {
				q.queryResult[namePath]["id"] = value
			}
		} else {
			// Just a value.
			targetPath, key := path.splitLastPath()
			if q.isRepeated[currentPath] {
				switch t := q.queryResult[targetPath][key].(type) {
				case nil:
					x := make([]interface{}, 0)
					x = append(x, value)
					q.queryResult[targetPath][key] = x
				case []interface{}:
					q.queryResult[targetPath][key] = append(t, value)
				}

			} else {
				q.queryResult[targetPath][key] = value
			}
		}
	}

	return resultPaths
}
Beispiel #10
0
func TestLoadDatabase(t *testing.T) {
	tmpDir, err := ioutil.TempDir(os.TempDir(), "cayley_test")
	if err != nil {
		t.Fatalf("Could not create working directory: %v", err)
	}
	defer os.RemoveAll(tmpDir)
	t.Log(tmpDir)

	err = createNewLevelDB(tmpDir, nil)
	if err != nil {
		t.Fatal("Failed to create LevelDB database.")
	}

	qs, err := newQuadStore(tmpDir, nil)
	if qs == nil || err != nil {
		t.Error("Failed to create leveldb QuadStore.")
	}

	w, _ := writer.NewSingleReplication(qs, nil)
	err = w.AddQuad(quad.MakeRaw(
		"Something",
		"points_to",
		"Something Else",
		"context",
	))
	if err != nil {
		t.Fatal("Failed to add quad:", err)
	}
	for _, pq := range []string{"Something", "points_to", "Something Else", "context"} {
		if got := quad.StringOf(qs.NameOf(qs.ValueOf(quad.Raw(pq)))); got != pq {
			t.Errorf("Failed to roundtrip %q, got:%q expect:%q", pq, got, pq)
		}
	}
	if s := qs.Size(); s != 1 {
		t.Errorf("Unexpected quadstore size, got:%d expect:1", s)
	}
	qs.Close()

	err = createNewLevelDB(tmpDir, nil)
	if err != graph.ErrDatabaseExists {
		t.Fatal("Failed to create LevelDB database.")
	}
	qs, err = newQuadStore(tmpDir, nil)
	if qs == nil || err != nil {
		t.Error("Failed to create leveldb QuadStore.")
	}
	w, _ = writer.NewSingleReplication(qs, nil)

	ts2, didConvert := qs.(*QuadStore)
	if !didConvert {
		t.Errorf("Could not convert from generic to LevelDB QuadStore")
	}

	//Test horizon
	horizon := qs.Horizon()
	if horizon.Int() != 1 {
		t.Errorf("Unexpected horizon value, got:%d expect:1", horizon.Int())
	}

	w.AddQuadSet(graphtest.MakeQuadSet())
	if s := qs.Size(); s != 12 {
		t.Errorf("Unexpected quadstore size, got:%d expect:12", s)
	}
	if s := ts2.SizeOf(qs.ValueOf(quad.Raw("B"))); s != 5 {
		t.Errorf("Unexpected quadstore size, got:%d expect:5", s)
	}
	horizon = qs.Horizon()
	if horizon.Int() != 12 {
		t.Errorf("Unexpected horizon value, got:%d expect:12", horizon.Int())
	}

	w.RemoveQuad(quad.MakeRaw(
		"A",
		"follows",
		"B",
		"",
	))
	if s := qs.Size(); s != 11 {
		t.Errorf("Unexpected quadstore size after RemoveQuad, got:%d expect:11", s)
	}
	if s := ts2.SizeOf(qs.ValueOf(quad.Raw("B"))); s != 4 {
		t.Errorf("Unexpected quadstore size, got:%d expect:4", s)
	}

	qs.Close()
}
Beispiel #11
0
func quadValueToString(v quad.Value) string {
	if s, ok := v.(quad.String); ok {
		return string(s)
	}
	return quad.StringOf(v)
}
Beispiel #12
0
func TestLoadTypedQuads(t testing.TB, gen DatabaseFunc, conf *Config) {
	qs, opts, closer := gen(t)
	defer closer()

	w := MakeWriter(t, qs, opts)

	values := []quad.Value{
		quad.BNode("A"), quad.IRI("name"), quad.String("B"), quad.IRI("graph"),
		quad.IRI("B"), quad.Raw("<type>"),
		quad.TypedString{Value: "10", Type: "int"},
		quad.LangString{Value: "value", Lang: "en"},
		quad.Int(-123456789),
		quad.Float(-12345e-6),
		quad.Bool(true),
		quad.Time(time.Now()),
	}

	err := w.AddQuadSet([]quad.Quad{
		{values[0], values[1], values[2], values[3]},
		{values[4], values[5], values[6], nil},
		{values[4], values[5], values[7], nil},
		{values[0], values[1], values[8], nil},
		{values[0], values[1], values[9], nil},
		{values[0], values[1], values[10], nil},
		{values[0], values[1], values[11], nil},
	})
	require.Nil(t, err)
	for _, pq := range values {
		got := qs.NameOf(qs.ValueOf(pq))
		if !conf.UnTyped {
			if pt, ok := pq.(quad.Time); ok {
				var trim int64
				if conf.TimeInMcs {
					trim = 1000
				} else if conf.TimeInMs {
					trim = 1000000
				}
				if trim > 0 {
					tm := time.Time(pt)
					seconds := tm.Unix()
					nanos := int64(tm.Sub(time.Unix(seconds, 0)))
					if conf.TimeRound {
						nanos = (nanos/trim + ((nanos/(trim/10))%10)/5) * trim
					} else {
						nanos = (nanos / trim) * trim
					}
					pq = quad.Time(time.Unix(seconds, nanos).UTC())
				}
			}
			if eq, ok := pq.(quad.Equaler); ok {
				assert.True(t, eq.Equal(got), "Failed to roundtrip %q (%T), got %q (%T)", pq, pq, got, got)
			} else {
				assert.Equal(t, pq, got, "Failed to roundtrip %q (%T)", pq, pq)
				if !conf.NoHashes {
					assert.Equal(t, pq, qs.NameOf(qs.ValueOf(quad.Raw(pq.String()))), "Failed to exchange raw value %q (%T)", pq, pq)
				}
			}
		} else {
			assert.Equal(t, quad.StringOf(pq), quad.StringOf(got), "Failed to roundtrip raw %q (%T)", pq, pq)
		}
	}
	require.Equal(t, int64(7), qs.Size(), "Unexpected quadstore size")
}