func (w *Writer) WriteQuad(q quad.Quad) error {
if w.err != nil {
return w.err
}
if !w.written {
if _, err := w.w.Write([]byte(header)); err != nil {
return err
}
w.written = true
w.nodes = make(map[string]int)
}
s := w.writeNode(quad.StringOf(q.Subject))
o := w.writeNode(quad.StringOf(q.Object))
if w.err != nil {
return w.err
}
_, w.err = fmt.Fprintf(w.w, "\t\t<edge source=\"n%d\" target=\"n%d\"><data key=\"d1\">", s, o)
if w.err != nil {
return w.err
}
if w.err = xml.EscapeText(w.w, []byte(quad.StringOf(q.Predicate))); w.err != nil {
return w.err
}
_, w.err = w.w.Write([]byte("</data></edge>\n"))
return w.err
}
func toTerm(qv quad.Value) gojsonld.Term {
if qv == nil {
return nil
}
switch v := qv.(type) {
case quad.IRI:
return gojsonld.NewResource(string(v))
case quad.BNode:
return gojsonld.NewBlankNode(string(v))
case quad.TypedString:
return gojsonld.NewLiteralWithDatatype(
string(v.Value),
gojsonld.NewResource(string(v.Type)),
)
case quad.LangString:
return gojsonld.NewLiteralWithLanguageAndDatatype(
string(v.Value),
string(v.Lang),
gojsonld.NewResource(gojsonld.XSD_STRING),
)
case quad.String:
return gojsonld.NewLiteralWithDatatype(
string(v),
gojsonld.NewResource(gojsonld.XSD_STRING),
)
}
t, err := turtle.Parse(quad.StringOf(qv))
if err != nil {
return gojsonld.NewLiteralWithDatatype(
string(quad.StringOf(qv)),
gojsonld.NewResource(gojsonld.XSD_STRING),
)
}
switch v := t.(type) {
case turtle.IRI:
return gojsonld.NewResource(string(v))
case turtle.BlankNode:
return gojsonld.NewBlankNode(string(v))
case turtle.String:
return gojsonld.NewLiteralWithDatatype(
string(v),
gojsonld.NewResource(gojsonld.XSD_STRING),
)
case turtle.LangString:
return gojsonld.NewLiteralWithLanguageAndDatatype(
string(v.Value),
v.Lang,
gojsonld.NewResource(gojsonld.XSD_STRING),
)
case turtle.TypedString:
return gojsonld.NewLiteralWithDatatype(
string(v.Value),
gojsonld.NewResource(string(v.Type)),
)
default:
return gojsonld.NewLiteralWithDatatype(quad.StringOf(qv), gojsonld.NewResource(gojsonld.XSD_STRING))
}
}
func (enc *Encoder) WriteQuad(q quad.Quad) error {
enc.writeValue(quad.StringOf(q.Subject))
enc.writeValue(quad.StringOf(q.Predicate))
enc.writeValue(quad.StringOf(q.Object))
if q.Label != nil {
enc.writeValue(quad.StringOf(q.Label))
}
if enc.err != nil {
return enc.err
}
_, enc.err = enc.w.Write([]byte(".\n"))
return enc.err
}
func (wk *worker) tagsToValueMap(m map[string]graph.Value) map[string]string {
outputMap := make(map[string]string)
for k, v := range m {
outputMap[k] = quad.StringOf(wk.qs.NameOf(v))
}
return outputMap
}
func (w *Writer) WriteQuad(q quad.Quad) error {
if w.err != nil {
return w.err
}
if !w.written {
if _, err := w.w.Write([]byte(header)); err != nil {
return err
}
w.written = true
w.nodes = make(map[string]int)
}
s := w.writeNode(quad.StringOf(q.Subject))
o := w.writeNode(quad.StringOf(q.Object))
if w.err != nil {
return w.err
}
_, w.err = fmt.Fprintf(w.w, "\tedge [ source %d target %d label %s ]\n",
s, o, escape(quad.StringOf(q.Predicate)))
return w.err
}
// Here's the non-boilerplate part of the ValueComparison iterator. Given a value
// and our operator, determine whether or not we meet the requirement.
func (it *Comparison) doComparison(val graph.Value) bool {
qval := it.qs.NameOf(val)
switch cVal := it.val.(type) {
case quad.Raw:
return RunStrOp(quad.StringOf(qval), it.op, string(cVal))
case quad.Int:
if cVal2, ok := qval.(quad.Int); ok {
return RunIntOp(cVal2, it.op, cVal)
}
return false
case quad.Float:
if cVal2, ok := qval.(quad.Float); ok {
return RunFloatOp(cVal2, it.op, cVal)
}
return false
case quad.String:
if cVal2, ok := qval.(quad.String); ok {
return RunStrOp(string(cVal2), it.op, string(cVal))
}
return false
case quad.BNode:
if cVal2, ok := qval.(quad.BNode); ok {
return RunStrOp(string(cVal2), it.op, string(cVal))
}
return false
case quad.IRI:
if cVal2, ok := qval.(quad.IRI); ok {
return RunStrOp(string(cVal2), it.op, string(cVal))
}
return false
case quad.Time:
if cVal2, ok := qval.(quad.Time); ok {
return RunTimeOp(time.Time(cVal2), it.op, time.Time(cVal))
}
return false
default:
return RunStrOp(quad.StringOf(qval), it.op, quad.StringOf(it.val))
}
}
func (w *Writer) WriteQuad(q quad.Quad) error {
graph := quad.StringOf(q.Label)
if graph == "" {
graph = "@default"
}
g := w.ds.Graphs[graph]
g = append(g, gojsonld.NewTriple(
toTerm(q.Subject),
toTerm(q.Predicate),
toTerm(q.Object),
))
w.ds.Graphs[graph] = g
return nil
}
func TestLoadOneQuad(t testing.TB, gen DatabaseFunc) {
qs, opts, closer := gen(t)
defer closer()
w := MakeWriter(t, qs, opts)
err := w.AddQuad(quad.Make(
"Something",
"points_to",
"Something Else",
"context",
))
require.Nil(t, err)
for _, pq := range []string{"Something", "points_to", "Something Else", "context"} {
got := quad.StringOf(qs.NameOf(qs.ValueOf(quad.Raw(pq))))
require.Equal(t, pq, got, "Failed to roundtrip %q", pq)
}
require.Equal(t, int64(1), qs.Size(), "Unexpected quadstore size")
}
func (qs *QuadStore) AddDelta(d graph.Delta) error {
if _, exists := qs.indexOf(d.Quad); exists {
return graph.ErrQuadExists
}
qs.logmu.Lock()
qid := qs.nextQuadID
qs.log = append(qs.log, LogEntry{
ID: d.ID.Int(),
Quad: d.Quad,
Action: d.Action,
Timestamp: d.Timestamp})
qs.size++
qs.nextQuadID++
qs.logmu.Unlock()
for dir := quad.Subject; dir <= quad.Label; dir++ {
sid := d.Quad.Get(dir)
if dir == quad.Label && sid == nil {
continue
}
ssid := quad.StringOf(sid)
qs.idmu.RLock()
id, ok := qs.idMap[ssid]
qs.idmu.RUnlock()
if !ok {
qs.idmu.Lock()
id, ok = qs.idMap[ssid]
if !ok {
id = qs.nextID
qs.idMap[ssid] = qs.nextID
qs.revIDMap[qs.nextID] = sid
qs.nextID++
}
qs.idmu.Unlock()
}
qs.index.Tree(dir, id).Set(qid)
}
// TODO(barakmich): Add VIP indexing
return nil
}
func NewIterator(qs *QuadStore, k string, d quad.Direction, val graph.Value) *Iterator {
t := val.(*Token)
if t == nil {
glog.Error("Token == nil")
}
if t.Kind != nodeKind {
glog.Error("Cannot create an iterator from a non-node value")
return &Iterator{done: true}
}
if k != nodeKind && k != quadKind {
glog.Error("Cannot create iterator for unknown kind")
return &Iterator{done: true}
}
if qs.context == nil {
glog.Error("Cannot create iterator without a valid context")
return &Iterator{done: true}
}
name := quad.StringOf(qs.NameOf(t))
// The number of references to this node is held in the nodes entity
key := qs.createKeyFromToken(t)
foundNode := new(NodeEntry)
err := datastore.Get(qs.context, key, foundNode)
if err != nil && err != datastore.ErrNoSuchEntity {
glog.Errorf("Error: %v", err)
return &Iterator{done: true}
}
size := foundNode.Size
return &Iterator{
uid: iterator.NextUID(),
name: name,
dir: d,
qs: qs,
size: size,
isAll: false,
kind: k,
hash: t.Hash,
done: false,
}
}
func runQueryGetTag(rec func(), g []quad.Quad, query string, tag string) []string {
js := makeTestSession(g)
c := make(chan interface{}, 1)
go func() {
defer rec()
js.Execute(query, c, -1)
}()
var results []string
for res := range c {
data := res.(*Result)
if data.val == nil {
val := data.actualResults[tag]
if val != nil {
results = append(results, quad.StringOf(js.qs.NameOf(val)))
}
}
}
return results
}
func toStrings(objs []interface{}) []string {
if len(objs) == 0 {
return nil
}
var out = make([]string, 0, len(objs))
for _, o := range objs {
switch v := o.(type) {
case string:
out = append(out, v)
case quad.Value:
out = append(out, quad.StringOf(v))
case []string:
out = append(out, v...)
case []interface{}:
out = append(out, toStrings(v)...)
default:
panic(fmt.Errorf("expected string, got: %T", o))
}
}
return out
}
func (wk *worker) runIteratorToArrayNoTags(it graph.Iterator, limit int) []string {
output := make([]string, 0)
n := 0
it, _ = it.Optimize()
for {
select {
case <-wk.kill:
return nil
default:
}
if !graph.Next(it) {
break
}
output = append(output, quad.StringOf(wk.qs.NameOf(it.Result())))
n++
if limit >= 0 && n >= limit {
break
}
}
it.Close()
return output
}
func (qs *QuadStore) indexOf(t quad.Quad) (int64, bool) {
min := maxInt
var tree *Tree
for d := quad.Subject; d <= quad.Label; d++ {
sid := t.Get(d)
if d == quad.Label && sid == nil {
continue
}
qs.idmu.RLock()
id, ok := qs.idMap[quad.StringOf(sid)]
qs.idmu.RUnlock()
// If we've never heard about a node, it must not exist
if !ok {
return 0, false
}
index, ok := qs.index.Get(d, id)
if !ok {
// If it's never been indexed in this direction, it can't exist.
return 0, false
}
if l := index.Len(); l < min {
min, tree = l, index
}
}
it := NewIterator(tree, qs, 0, nil)
for it.Next() {
qs.logmu.RLock()
l := qs.log[it.result]
qs.logmu.RUnlock()
if t == l.Quad {
return it.result, true
}
}
return 0, false
}
func TestLoadTypedQuads(t testing.TB, gen DatabaseFunc, conf *Config) {
qs, opts, closer := gen(t)
defer closer()
w := MakeWriter(t, qs, opts)
values := []quad.Value{
quad.BNode("A"), quad.IRI("name"), quad.String("B"), quad.IRI("graph"),
quad.IRI("B"), quad.Raw("<type>"),
quad.TypedString{Value: "10", Type: "int"},
quad.LangString{Value: "value", Lang: "en"},
quad.Int(-123456789),
quad.Float(-12345e-6),
quad.Bool(true),
quad.Time(time.Now()),
}
err := w.AddQuadSet([]quad.Quad{
{values[0], values[1], values[2], values[3]},
{values[4], values[5], values[6], nil},
{values[4], values[5], values[7], nil},
{values[0], values[1], values[8], nil},
{values[0], values[1], values[9], nil},
{values[0], values[1], values[10], nil},
{values[0], values[1], values[11], nil},
})
require.Nil(t, err)
for _, pq := range values {
got := qs.NameOf(qs.ValueOf(pq))
if !conf.UnTyped {
if pt, ok := pq.(quad.Time); ok {
var trim int64
if conf.TimeInMcs {
trim = 1000
} else if conf.TimeInMs {
trim = 1000000
}
if trim > 0 {
tm := time.Time(pt)
seconds := tm.Unix()
nanos := int64(tm.Sub(time.Unix(seconds, 0)))
if conf.TimeRound {
nanos = (nanos/trim + ((nanos/(trim/10))%10)/5) * trim
} else {
nanos = (nanos / trim) * trim
}
pq = quad.Time(time.Unix(seconds, nanos).UTC())
}
}
if eq, ok := pq.(quad.Equaler); ok {
assert.True(t, eq.Equal(got), "Failed to roundtrip %q (%T), got %q (%T)", pq, pq, got, got)
} else {
assert.Equal(t, pq, got, "Failed to roundtrip %q (%T)", pq, pq)
if !conf.NoHashes {
assert.Equal(t, pq, qs.NameOf(qs.ValueOf(quad.Raw(pq.String()))), "Failed to exchange raw value %q (%T)", pq, pq)
}
}
} else {
assert.Equal(t, quad.StringOf(pq), quad.StringOf(got), "Failed to roundtrip raw %q (%T)", pq, pq)
}
}
require.Equal(t, int64(7), qs.Size(), "Unexpected quadstore size")
}
func (q *Query) treeifyResult(tags map[string]graph.Value) map[ResultPath]string {
// Transform the map into something a little more interesting.
results := make(map[Path]string)
for k, v := range tags {
if v == nil {
continue
}
results[Path(k)] = quad.StringOf(q.ses.qs.NameOf(v))
}
resultPaths := make(map[ResultPath]string)
for k, v := range results {
resultPaths[k.ToResultPathFromMap(results)] = v
}
paths := make([]ResultPath, 0, len(resultPaths))
for path := range resultPaths {
paths = append(paths, path)
}
sort.Sort(byRecordLength(paths))
// Build Structure
for _, path := range paths {
currentPath := path.getPath()
value := resultPaths[path]
namePath := path.AppendValue(value)
if _, ok := q.queryResult[namePath]; !ok {
targetPath, key := path.splitLastPath()
if path == "" {
targetPath, key = "", value
if _, ok := q.queryResult[""][value]; !ok {
q.resultOrder = append(q.resultOrder, value)
}
}
if _, ok := q.queryStructure[currentPath]; ok {
// If there's substructure, then copy that in.
newStruct := q.copyPathStructure(currentPath)
if q.isRepeated[currentPath] && currentPath != "" {
switch t := q.queryResult[targetPath][key].(type) {
case nil:
x := make([]interface{}, 0)
x = append(x, newStruct)
q.queryResult[targetPath][key] = x
q.queryResult[namePath] = newStruct
case []interface{}:
q.queryResult[targetPath][key] = append(t, newStruct)
q.queryResult[namePath] = newStruct
}
} else {
q.queryResult[namePath] = newStruct
q.queryResult[targetPath][key] = newStruct
}
}
}
}
// Fill values
for _, path := range paths {
currentPath := path.getPath()
value, ok := resultPaths[path]
if !ok {
continue
}
namePath := path.AppendValue(value)
if _, ok := q.queryStructure[currentPath]; ok {
// We're dealing with ids.
if _, ok := q.queryResult[namePath]["id"]; ok {
q.queryResult[namePath]["id"] = value
}
} else {
// Just a value.
targetPath, key := path.splitLastPath()
if q.isRepeated[currentPath] {
switch t := q.queryResult[targetPath][key].(type) {
case nil:
x := make([]interface{}, 0)
x = append(x, value)
q.queryResult[targetPath][key] = x
case []interface{}:
q.queryResult[targetPath][key] = append(t, value)
}
} else {
q.queryResult[targetPath][key] = value
}
}
}
return resultPaths
}
func (qs *QuadStore) ValueOf(name quad.Value) graph.Value {
qs.idmu.RLock()
v := qs.idMap[quad.StringOf(name)]
qs.idmu.RUnlock()
return iterator.Int64Node(v)
}
func TestLoadDatabase(t *testing.T) {
tmpDir, err := ioutil.TempDir(os.TempDir(), "cayley_test")
if err != nil {
t.Fatalf("Could not create working directory: %v", err)
}
defer os.RemoveAll(tmpDir)
t.Log(tmpDir)
err = createNewLevelDB(tmpDir, nil)
if err != nil {
t.Fatal("Failed to create LevelDB database.")
}
qs, err := newQuadStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb QuadStore.")
}
w, _ := writer.NewSingleReplication(qs, nil)
err = w.WriteQuad(quad.Make(
"Something",
"points_to",
"Something Else",
"context",
))
if err != nil {
t.Fatal("Failed to add quad:", err)
}
for _, pq := range []string{"Something", "points_to", "Something Else", "context"} {
if got := quad.StringOf(qs.NameOf(qs.ValueOf(quad.Raw(pq)))); got != pq {
t.Errorf("Failed to roundtrip %q, got:%q expect:%q", pq, got, pq)
}
}
if s := qs.Size(); s != 1 {
t.Errorf("Unexpected quadstore size, got:%d expect:1", s)
}
qs.Close()
err = createNewLevelDB(tmpDir, nil)
if err != graph.ErrDatabaseExists {
t.Fatal("Failed to create LevelDB database.")
}
qs, err = newQuadStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb QuadStore.")
}
w, _ = writer.NewSingleReplication(qs, nil)
ts2, didConvert := qs.(*QuadStore)
if !didConvert {
t.Errorf("Could not convert from generic to LevelDB QuadStore")
}
//Test horizon
horizon := qs.Horizon()
if horizon.Int() != 1 {
t.Errorf("Unexpected horizon value, got:%d expect:1", horizon.Int())
}
w.WriteQuads(graphtest.MakeQuadSet())
if s := qs.Size(); s != 12 {
t.Errorf("Unexpected quadstore size, got:%d expect:12", s)
}
if s := ts2.SizeOf(qs.ValueOf(quad.Raw("B"))); s != 5 {
t.Errorf("Unexpected quadstore size, got:%d expect:5", s)
}
horizon = qs.Horizon()
if horizon.Int() != 12 {
t.Errorf("Unexpected horizon value, got:%d expect:12", horizon.Int())
}
w.RemoveQuad(quad.Make(
"A",
"follows",
"B",
"",
))
if s := qs.Size(); s != 11 {
t.Errorf("Unexpected quadstore size after RemoveQuad, got:%d expect:11", s)
}
if s := ts2.SizeOf(qs.ValueOf(quad.Raw("B"))); s != 4 {
t.Errorf("Unexpected quadstore size, got:%d expect:4", s)
}
qs.Close()
}