func TestNewGraph(t *testing.T) {
var ex uint64
ex = 101
sg, err := NewGraph(ex, "")
if err != nil {
t.Error(err)
}
uo := flatbuffers.GetUOffsetT(sg.result)
r := new(task.Result)
r.Init(sg.result, uo)
if r.UidmatrixLength() != 1 {
t.Errorf("Expected length 1. Got: %v", r.UidmatrixLength())
}
var ul task.UidList
if ok := r.Uidmatrix(&ul, 0); !ok {
t.Errorf("Unable to parse uidlist at index 0")
}
if ul.UidsLength() != 1 {
t.Errorf("Expected length 1. Got: %v", ul.UidsLength())
}
if ul.Uids(0) != ex {
t.Errorf("Expected uid: %v. Got: %v", ex, ul.Uids(0))
}
}
func checkName(t *testing.T, r *task.Result, idx int, expected string) {
var tv task.Value
if ok := r.Values(&tv, idx); !ok {
t.Error("Unable to retrieve value")
}
var iname interface{}
if err := posting.ParseValue(&iname, tv.ValBytes()); err != nil {
t.Error(err)
}
name := iname.(string)
if name != expected {
t.Errorf("Expected: %v. Got: %v", expected, name)
}
}
func check(r *task.Result, idx int, expected []uint64) error {
var m task.UidList
if ok := r.Uidmatrix(&m, idx); !ok {
return fmt.Errorf("Unable to retrieve uidlist")
}
if m.UidsLength() != len(expected) {
return fmt.Errorf("Expected length: %v. Got: %v",
len(expected), m.UidsLength())
}
for i, uid := range expected {
if m.Uids(i) != uid {
return fmt.Errorf("Uid mismatch at index: %v. Expected: %v. Got: %v",
i, uid, m.Uids(i))
}
}
return nil
}
func TestNewGraph(t *testing.T) {
var ex uint64
ex = 101
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
t.Error(err)
return
}
ps := new(store.Store)
ps.Init(dir)
sg, err := newGraph(ex, "", ps)
if err != nil {
t.Error(err)
}
worker.Init(ps)
uo := flatbuffers.GetUOffsetT(sg.result)
r := new(task.Result)
r.Init(sg.result, uo)
if r.UidmatrixLength() != 1 {
t.Errorf("Expected length 1. Got: %v", r.UidmatrixLength())
}
var ul task.UidList
if ok := r.Uidmatrix(&ul, 0); !ok {
t.Errorf("Unable to parse uidlist at index 0")
}
if ul.UidsLength() != 1 {
t.Errorf("Expected length 1. Got: %v", ul.UidsLength())
}
if ul.Uids(0) != ex {
t.Errorf("Expected uid: %v. Got: %v", ex, ul.Uids(0))
}
}
func sortedUniqueUids(r *task.Result) (sorted []uint64, rerr error) {
// Let's serialize the matrix of uids in result to a
// sorted unique list of uids.
h := &x.Uint64Heap{}
heap.Init(h)
channels := make([]*ListChannel, r.UidmatrixLength())
for i := 0; i < r.UidmatrixLength(); i++ {
tlist := new(task.UidList)
if ok := r.Uidmatrix(tlist, i); !ok {
return sorted, fmt.Errorf("While parsing Uidmatrix")
}
if tlist.UidsLength() > 0 {
e := x.Elem{
Uid: tlist.Uids(0),
Idx: i,
}
heap.Push(h, e)
}
channels[i] = &ListChannel{TList: tlist, Idx: 1}
}
// The resulting list of uids will be stored here.
sorted = make([]uint64, 100)
sorted = sorted[:0]
var last uint64
last = 0
// Itearate over the heap.
for h.Len() > 0 {
me := (*h)[0] // Peek at the top element in heap.
if me.Uid != last {
sorted = append(sorted, me.Uid) // Add if unique.
last = me.Uid
}
lc := channels[me.Idx]
if lc.Idx >= lc.TList.UidsLength() {
heap.Pop(h)
} else {
uid := lc.TList.Uids(lc.Idx)
lc.Idx += 1
me.Uid = uid
(*h)[0] = me
heap.Fix(h, 0) // Faster than Pop() followed by Push().
}
}
return sorted, nil
}
func ProcessGraph(sg *SubGraph, rch chan error) {
var err error
if len(sg.query) > 0 && sg.Attr != "_root_" {
// This task execution would go over the wire in later versions.
sg.result, err = posting.ProcessTask(sg.query)
if err != nil {
x.Err(glog, err).Error("While processing task.")
rch <- err
return
}
}
uo := flatbuffers.GetUOffsetT(sg.result)
r := new(task.Result)
r.Init(sg.result, uo)
sorted, err := sortedUniqueUids(r)
if err != nil {
x.Err(glog, err).Error("While processing task.")
rch <- err
return
}
if len(sorted) == 0 {
// Looks like we're done here.
if len(sg.Children) > 0 {
glog.Debugf("Have some children but no results. Life got cut short early."+
"Current attribute: %q", sg.Attr)
} else {
glog.Debugf("No more things to process for Attr: %v", sg.Attr)
}
rch <- nil
return
}
// Let's execute it in a tree fashion. Each SubGraph would break off
// as many goroutines as it's children; which would then recursively
// do the same thing.
// Buffered channel to ensure no-blockage.
childchan := make(chan error, len(sg.Children))
for i := 0; i < len(sg.Children); i++ {
child := sg.Children[i]
child.query = createTaskQuery(child.Attr, sorted)
go ProcessGraph(child, childchan)
}
// Now get all the results back.
for i := 0; i < len(sg.Children); i++ {
err = <-childchan
glog.WithFields(logrus.Fields{
"num_children": len(sg.Children),
"index": i,
"attr": sg.Children[i].Attr,
"err": err,
}).Debug("Reply from child")
if err != nil {
x.Err(glog, err).Error("While processing child task.")
rch <- err
return
}
}
rch <- nil
}
func postTraverse(g *SubGraph) (result map[uint64]interface{}, rerr error) {
if len(g.query) == 0 {
return result, nil
}
result = make(map[uint64]interface{})
// Get results from all children first.
cResult := make(map[uint64]interface{})
for _, child := range g.Children {
m, err := postTraverse(child)
if err != nil {
x.Err(glog, err).Error("Error while traversal")
return result, err
}
// Merge results from all children, one by one.
for k, v := range m {
if val, present := cResult[k]; !present {
cResult[k] = v
} else {
cResult[k] = mergeInterfaces(val, v)
}
}
}
// Now read the query and results at current node.
uo := flatbuffers.GetUOffsetT(g.query)
q := new(task.Query)
q.Init(g.query, uo)
ro := flatbuffers.GetUOffsetT(g.result)
r := new(task.Result)
r.Init(g.result, ro)
if q.UidsLength() != r.UidmatrixLength() {
glog.Fatal("Result uidmatrixlength: %v. Query uidslength: %v",
r.UidmatrixLength(), q.UidsLength())
}
if q.UidsLength() != r.ValuesLength() {
glog.Fatalf("Result valuelength: %v. Query uidslength: %v",
r.ValuesLength(), q.UidsLength())
}
var ul task.UidList
for i := 0; i < r.UidmatrixLength(); i++ {
if ok := r.Uidmatrix(&ul, i); !ok {
return result, fmt.Errorf("While parsing UidList")
}
l := make([]interface{}, ul.UidsLength())
for j := 0; j < ul.UidsLength(); j++ {
uid := ul.Uids(j)
m := make(map[string]interface{})
m["_uid_"] = fmt.Sprintf("%#x", uid)
if ival, present := cResult[uid]; !present {
l[j] = m
} else {
l[j] = mergeInterfaces(m, ival)
}
}
if len(l) > 0 {
m := make(map[string]interface{})
m[g.Attr] = l
result[q.Uids(i)] = m
}
}
var tv task.Value
for i := 0; i < r.ValuesLength(); i++ {
if ok := r.Values(&tv, i); !ok {
return result, fmt.Errorf("While parsing value")
}
var ival interface{}
if err := posting.ParseValue(&ival, tv.ValBytes()); err != nil {
return result, err
}
if ival == nil {
continue
}
if pval, present := result[q.Uids(i)]; present {
glog.WithField("prev", pval).
WithField("_uid_", q.Uids(i)).
WithField("new", ival).
Fatal("Previous value detected.")
}
m := make(map[string]interface{})
m["_uid_"] = fmt.Sprintf("%#x", q.Uids(i))
glog.WithFields(logrus.Fields{
"_uid_": q.Uids(i),
"val": ival,
}).Debug("Got value")
m[g.Attr] = ival
result[q.Uids(i)] = m
}
return result, nil
}
// processTask processes the query, accumulates and returns the result.
func processTask(q *task.Query) (*task.Result, error) {
attr := q.Attr
useFunc := len(q.SrcFunc) != 0
var n int
var tokens []string
var geoQuery *geo.QueryData
var err error
var intersectDest bool
var ineqValue types.Value
var ineqValueToken string
var isGeq, isLeq bool
if useFunc {
f := q.SrcFunc[0]
isGeq = f == "geq"
isLeq = f == "leq"
switch {
case isGeq:
fallthrough
case isLeq:
if len(q.SrcFunc) != 2 {
return nil, x.Errorf("Function requires 2 arguments, but got %d %v",
len(q.SrcFunc), q.SrcFunc)
}
ineqValue, err = getValue(attr, q.SrcFunc[1])
if err != nil {
return nil, err
}
// Tokenizing RHS value of inequality.
ineqTokens, err := posting.IndexTokens(attr, ineqValue)
if err != nil {
return nil, err
}
if len(ineqTokens) != 1 {
return nil, x.Errorf("Expected only 1 token but got: %v", ineqTokens)
}
ineqValueToken = ineqTokens[0]
// Get tokens geq / leq ineqValueToken.
tokens, err = getInequalityTokens(attr, ineqValueToken, isGeq)
if err != nil {
return nil, err
}
case geo.IsGeoFunc(q.SrcFunc[0]):
// For geo functions, we get extra information used for filtering.
tokens, geoQuery, err = geo.GetTokens(q.SrcFunc)
if err != nil {
return nil, err
}
default:
tokens, err = getTokens(q.SrcFunc)
if err != nil {
return nil, err
}
intersectDest = (strings.ToLower(q.SrcFunc[0]) == "allof")
}
n = len(tokens)
} else {
n = len(q.Uids)
}
var out task.Result
for i := 0; i < n; i++ {
var key []byte
if useFunc {
key = x.IndexKey(attr, tokens[i])
} else {
key = x.DataKey(attr, q.Uids[i])
}
// Get or create the posting list for an entity, attribute combination.
pl, decr := posting.GetOrCreate(key)
defer decr()
// If a posting list contains a value, we store that or else we store a nil
// byte so that processing is consistent later.
vbytes, vtype, err := pl.Value()
newValue := &task.Value{ValType: uint32(vtype)}
if err == nil {
newValue.Val = vbytes
} else {
newValue.Val = x.Nilbyte
}
out.Values = append(out.Values, newValue)
if q.DoCount {
out.Counts = append(out.Counts, uint32(pl.Length(0)))
// Add an empty UID list to make later processing consistent
out.UidMatrix = append(out.UidMatrix, &emptyUIDList)
continue
}
// The more usual case: Getting the UIDs.
opts := posting.ListOptions{
AfterUID: uint64(q.AfterUid),
}
// If we have srcFunc and Uids, it means its a filter. So we intersect.
if useFunc && len(q.Uids) > 0 {
opts.Intersect = &task.List{Uids: q.Uids}
}
out.UidMatrix = append(out.UidMatrix, pl.Uids(opts))
}
if (isGeq || isLeq) && len(tokens) > 0 && ineqValueToken == tokens[0] {
// Need to evaluate inequality for entries in the first bucket.
typ := schema.TypeOf(attr)
if typ == nil || !typ.IsScalar() {
return nil, x.Errorf("Attribute not scalar: %s %v", attr, typ)
}
scalarType := typ.(types.Scalar)
x.AssertTrue(len(out.UidMatrix) > 0)
// Filter the first row of UidMatrix. Since ineqValue != nil, we may
// assume that ineqValue is equal to the first token found in TokensTable.
algo.ApplyFilter(out.UidMatrix[0], func(uid uint64, i int) bool {
key := x.DataKey(attr, uid)
sv := getPostingValue(key, scalarType)
if sv == nil {
return false
}
if isGeq {
return !scalarType.Less(*sv, ineqValue)
}
return !scalarType.Less(ineqValue, *sv)
})
}
// If geo filter, do value check for correctness.
var values []*task.Value
if geoQuery != nil {
uids := algo.MergeSorted(out.UidMatrix)
for _, uid := range uids.Uids {
key := x.DataKey(attr, uid)
pl, decr := posting.GetOrCreate(key)
vbytes, vtype, err := pl.Value()
newValue := &task.Value{ValType: uint32(vtype)}
if err == nil {
newValue.Val = vbytes
} else {
newValue.Val = x.Nilbyte
}
values = append(values, newValue)
decr() // Decrement the reference count of the pl.
}
filtered := geo.FilterUids(uids, values, geoQuery)
for i := 0; i < len(out.UidMatrix); i++ {
out.UidMatrix[i] = algo.IntersectSorted([]*task.List{out.UidMatrix[i], filtered})
}
}
out.IntersectDest = intersectDest
return &out, nil
}
func TestProcessTask(t *testing.T) {
// logrus.SetLevel(logrus.DebugLevel)
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
t.Error(err)
return
}
defer os.RemoveAll(dir)
ps := new(store.Store)
ps.Init(dir)
clog := commit.NewLogger(dir, "mutations", 50<<20)
clog.Init()
defer clog.Close()
Init(ps, clog)
edge := x.DirectedEdge{
ValueId: 23,
Source: "author0",
Timestamp: time.Now(),
}
addEdge(t, edge, GetOrCreate(Key(10, "friend")))
addEdge(t, edge, GetOrCreate(Key(11, "friend")))
addEdge(t, edge, GetOrCreate(Key(12, "friend")))
edge.ValueId = 25
addEdge(t, edge, GetOrCreate(Key(12, "friend")))
edge.ValueId = 26
addEdge(t, edge, GetOrCreate(Key(12, "friend")))
edge.ValueId = 31
addEdge(t, edge, GetOrCreate(Key(10, "friend")))
addEdge(t, edge, GetOrCreate(Key(12, "friend")))
edge.Value = "photon"
addEdge(t, edge, GetOrCreate(Key(12, "friend")))
query := NewQuery("friend", []uint64{10, 11, 12})
result, err := ProcessTask(query)
if err != nil {
t.Error(err)
}
ro := flatbuffers.GetUOffsetT(result)
r := new(task.Result)
r.Init(result, ro)
if r.UidmatrixLength() != 3 {
t.Errorf("Expected 3. Got uidmatrix length: %v", r.UidmatrixLength())
}
if err := check(r, 0, []uint64{23, 31}); err != nil {
t.Error(err)
}
if err := check(r, 1, []uint64{23}); err != nil {
t.Error(err)
}
if err := check(r, 2, []uint64{23, 25, 26, 31}); err != nil {
t.Error(err)
}
if r.ValuesLength() != 3 {
t.Errorf("Expected 3. Got values length: %v", r.ValuesLength())
}
var tval task.Value
if ok := r.Values(&tval, 0); !ok {
t.Errorf("Unable to retrieve value")
}
if tval.ValLength() != 1 ||
tval.ValBytes()[0] != 0x00 {
t.Errorf("Invalid byte value at index 0")
}
if ok := r.Values(&tval, 1); !ok {
t.Errorf("Unable to retrieve value")
}
if tval.ValLength() != 1 ||
tval.ValBytes()[0] != 0x00 {
t.Errorf("Invalid byte value at index 0")
}
if ok := r.Values(&tval, 2); !ok {
t.Errorf("Unable to retrieve value")
}
var iout interface{}
if err := ParseValue(&iout, tval.ValBytes()); err != nil {
t.Error(err)
}
v := iout.(string)
if v != "photon" {
t.Errorf("Expected photon. Got: %q", v)
}
}
func checkSingleValue(t *testing.T, child *SubGraph,
attr string, value string) {
if child.Attr != attr || len(child.result) == 0 {
t.Error("Expected attr name with some result")
}
uo := flatbuffers.GetUOffsetT(child.result)
r := new(task.Result)
r.Init(child.result, uo)
if r.ValuesLength() != 1 {
t.Errorf("Expected value length 1. Got: %v", r.ValuesLength())
}
if r.UidmatrixLength() != 1 {
t.Errorf("Expected uidmatrix length 1. Got: %v", r.UidmatrixLength())
}
var ul task.UidList
if ok := r.Uidmatrix(&ul, 0); !ok {
t.Errorf("While parsing uidlist")
}
if ul.UidsLength() != 0 {
t.Error("Expected uids length 0. Got: %v", ul.UidsLength())
}
checkName(t, r, 0, value)
}
func TestProcessGraph(t *testing.T) {
dir, ps := populateGraph(t)
defer os.RemoveAll(dir)
// Alright. Now we have everything set up. Let's create the query.
query := `
{
me(_uid_: 0x01) {
friend {
name
}
name
gender
status
}
}
`
gq, _, err := gql.Parse(query)
if err != nil {
t.Error(err)
}
sg, err := ToSubGraph(gq, ps)
if err != nil {
t.Error(err)
}
ch := make(chan error)
go ProcessGraph(sg, ch, ps)
err = <-ch
if err != nil {
t.Error(err)
}
if len(sg.Children) != 4 {
t.Errorf("Expected len 4. Got: %v", len(sg.Children))
}
child := sg.Children[0]
if child.Attr != "friend" {
t.Errorf("Expected attr friend. Got: %v", child.Attr)
}
if len(child.result) == 0 {
t.Errorf("Expected some result.")
return
}
uo := flatbuffers.GetUOffsetT(child.result)
r := new(task.Result)
r.Init(child.result, uo)
if r.UidmatrixLength() != 1 {
t.Errorf("Expected 1 matrix. Got: %v", r.UidmatrixLength())
}
var ul task.UidList
if ok := r.Uidmatrix(&ul, 0); !ok {
t.Errorf("While parsing uidlist")
}
if ul.UidsLength() != 5 {
t.Errorf("Expected 5 friends. Got: %v", ul.UidsLength())
}
if ul.Uids(0) != 23 || ul.Uids(1) != 24 || ul.Uids(2) != 25 ||
ul.Uids(3) != 31 || ul.Uids(4) != 101 {
t.Errorf("Friend ids don't match")
}
if len(child.Children) != 1 || child.Children[0].Attr != "name" {
t.Errorf("Expected attr name")
}
child = child.Children[0]
uo = flatbuffers.GetUOffsetT(child.result)
r.Init(child.result, uo)
if r.ValuesLength() != 5 {
t.Errorf("Expected 5 names of 5 friends")
}
checkName(t, r, 0, "Rick Grimes")
checkName(t, r, 1, "Glenn Rhee")
checkName(t, r, 2, "Daryl Dixon")
checkName(t, r, 3, "Andrea")
{
var tv task.Value
if ok := r.Values(&tv, 4); !ok {
t.Error("Unable to retrieve value")
}
if tv.ValLength() != 1 || tv.ValBytes()[0] != 0x00 {
t.Error("Expected a null byte")
}
}
checkSingleValue(t, sg.Children[1], "name", "Michonne")
checkSingleValue(t, sg.Children[2], "gender", "female")
checkSingleValue(t, sg.Children[3], "status", "alive")
}