// ProcessTaskOverNetwork is used to process the query and get the result from
// the instance which stores posting list corresponding to the predicate in the
// query.
func ProcessTaskOverNetwork(ctx context.Context, q *task.Query) (*task.Result, error) {
attr := q.Attr
gid := group.BelongsTo(attr)
x.Trace(ctx, "attr: %v groupId: %v", attr, gid)
if groups().ServesGroup(gid) {
// No need for a network call, as this should be run from within this instance.
return processTask(q)
}
// Send this over the network.
// TODO: Send the request to multiple servers as described in Jeff Dean's talk.
addr := groups().AnyServer(gid)
pl := pools().get(addr)
conn, err := pl.Get()
if err != nil {
return &emptyResult, x.Wrapf(err, "ProcessTaskOverNetwork: while retrieving connection.")
}
defer pl.Put(conn)
x.Trace(ctx, "Sending request to %v", addr)
c := NewWorkerClient(conn)
reply, err := c.ServeTask(ctx, q)
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while calling Worker.ServeTask"))
return &emptyResult, err
}
x.Trace(ctx, "Reply from server. length: %v Addr: %v Attr: %v",
len(reply.UidMatrix), addr, attr)
return reply, nil
}
func addIndexMutation(ctx context.Context, attr, token string,
tokensTable *TokensTable, edge *task.DirectedEdge, del bool) {
key := x.IndexKey(attr, token)
plist, decr := GetOrCreate(key)
defer decr()
x.AssertTruef(plist != nil, "plist is nil [%s] %d %s",
token, edge.ValueId, edge.Attr)
if del {
_, err := plist.AddMutation(ctx, edge, Del)
if err != nil {
x.TraceError(ctx, x.Wrapf(err,
"Error deleting %s for attr %s entity %d: %v",
token, edge.Attr, edge.Entity))
}
indexLog.Printf("DEL [%s] [%d] OldTerm [%s]",
edge.Attr, edge.Entity, token)
} else {
_, err := plist.AddMutation(ctx, edge, Set)
if err != nil {
x.TraceError(ctx, x.Wrapf(err,
"Error adding %s for attr %s entity %d: %v",
token, edge.Attr, edge.Entity))
}
indexLog.Printf("SET [%s] [%d] NewTerm [%s]",
edge.Attr, edge.Entity, token)
tokensTable.Add(token)
}
}
func (w *Wal) HardState(gid uint32) (hd raftpb.HardState, rerr error) {
slice, err := w.wals.Get(w.hardStateKey(gid))
if err != nil || slice == nil {
return hd, x.Wrapf(err, "While getting hardstate")
}
rerr = x.Wrapf(hd.Unmarshal(slice.Data()), "While unmarshal hardstate")
slice.Free()
return
}
func (w *Wal) Snapshot(gid uint32) (snap raftpb.Snapshot, rerr error) {
slice, err := w.wals.Get(w.snapshotKey(gid))
if err != nil || slice == nil {
return snap, x.Wrapf(err, "While getting snapshot")
}
rerr = x.Wrapf(snap.Unmarshal(slice.Data()), "While unmarshal snapshot")
slice.Free()
return
}
func convertToEdges(ctx context.Context, nquads []rdf.NQuad) (mutationResult, error) {
var edges []*task.DirectedEdge
var mr mutationResult
newUids := make(map[string]uint64)
for _, nq := range nquads {
if strings.HasPrefix(nq.Subject, "_new_:") {
newUids[nq.Subject] = 0
} else if !strings.HasPrefix(nq.Subject, "_uid_:") {
uid, err := rdf.GetUid(nq.Subject)
x.Check(err)
newUids[nq.Subject] = uid
}
if len(nq.ObjectId) > 0 {
if strings.HasPrefix(nq.ObjectId, "_new_:") {
newUids[nq.ObjectId] = 0
} else if !strings.HasPrefix(nq.ObjectId, "_uid_:") {
uid, err := rdf.GetUid(nq.ObjectId)
x.Check(err)
newUids[nq.ObjectId] = uid
}
}
}
if len(newUids) > 0 {
if err := worker.AssignUidsOverNetwork(ctx, newUids); err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while GetOrAssignUidsOverNetwork"))
return mr, err
}
}
for _, nq := range nquads {
// Get edges from nquad using newUids.
edge, err := nq.ToEdgeUsing(newUids)
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while converting to edge: %v", nq))
return mr, err
}
edges = append(edges, edge)
}
resultUids := make(map[string]uint64)
// Strip out _new_: prefix from the keys.
for k, v := range newUids {
if strings.HasPrefix(k, "_new_:") {
resultUids[k[6:]] = v
}
}
mr = mutationResult{
edges: edges,
newUids: resultUids,
}
return mr, nil
}
// Store stores the snapshot, hardstate and entries for a given RAFT group.
func (w *Wal) Store(gid uint32, s raftpb.Snapshot, h raftpb.HardState, es []raftpb.Entry) error {
b := w.wals.NewWriteBatch()
defer b.Destroy()
if !raft.IsEmptySnap(s) {
data, err := s.Marshal()
if err != nil {
return x.Wrapf(err, "wal.Store: While marshal snapshot")
}
b.Put(w.snapshotKey(gid), data)
}
if !raft.IsEmptyHardState(h) {
data, err := h.Marshal()
if err != nil {
return x.Wrapf(err, "wal.Store: While marshal hardstate")
}
b.Put(w.hardStateKey(gid), data)
}
var t, i uint64
for _, e := range es {
t, i = e.Term, e.Index
data, err := e.Marshal()
if err != nil {
return x.Wrapf(err, "wal.Store: While marshal entry")
}
k := w.entryKey(gid, e.Term, e.Index)
b.Put(k, data)
}
// If we get no entries, then the default value of t and i would be zero. That would
// end up deleting all the previous valid raft entry logs. This check avoids that.
if t > 0 || i > 0 {
// Delete all keys above this index.
start := w.entryKey(gid, t, i+1)
prefix := w.prefix(gid)
itr := w.wals.NewIterator()
defer itr.Close()
for itr.Seek(start); itr.ValidForPrefix(prefix); itr.Next() {
b.Delete(itr.Key().Data())
}
}
err := w.wals.WriteBatch(b)
return x.Wrapf(err, "wal.Store: While WriteBatch")
}
// MutateOverNetwork checks which group should be running the mutations
// according to fingerprint of the predicate and sends it to that instance.
func MutateOverNetwork(ctx context.Context, m *task.Mutations) error {
mutationMap := make(map[uint32]*task.Mutations)
addToMutationMap(mutationMap, m.Set, set)
addToMutationMap(mutationMap, m.Del, del)
errors := make(chan error, len(mutationMap))
for gid, mu := range mutationMap {
proposeOrSend(ctx, gid, mu, errors)
}
// Wait for all the goroutines to reply back.
// We return if an error was returned or the parent called ctx.Done()
for i := 0; i < len(mutationMap); i++ {
select {
case err := <-errors:
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while running all mutations"))
return err
}
case <-ctx.Done():
return ctx.Err()
}
}
close(errors)
return nil
}
func convertToNQuad(ctx context.Context, mutation string) ([]rdf.NQuad, error) {
var nquads []rdf.NQuad
r := strings.NewReader(mutation)
reader := bufio.NewReader(r)
x.Trace(ctx, "Converting to NQuad")
var strBuf bytes.Buffer
var err error
for {
err = x.ReadLine(reader, &strBuf)
if err != nil {
break
}
ln := strings.Trim(strBuf.String(), " \t")
if len(ln) == 0 {
continue
}
nq, err := rdf.Parse(ln)
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while parsing RDF"))
return nquads, err
}
nquads = append(nquads, nq)
}
if err != io.EOF {
return nquads, err
}
return nquads, nil
}
// PopulateShard gets data for predicate pred from server with id serverId and
// writes it to RocksDB.
func populateShard(ctx context.Context, pl *pool, group uint32) (int, error) {
gkeys, err := generateGroup(group)
if err != nil {
return 0, x.Wrapf(err, "While generating keys group")
}
conn, err := pl.Get()
if err != nil {
return 0, err
}
defer pl.Put(conn)
c := NewWorkerClient(conn)
stream, err := c.PredicateData(context.Background(), gkeys)
if err != nil {
return 0, err
}
x.Trace(ctx, "Streaming data for group: %v", group)
kvs := make(chan *task.KV, 1000)
che := make(chan error)
go writeBatch(ctx, kvs, che)
// We can use count to check the number of posting lists returned in tests.
count := 0
for {
kv, err := stream.Recv()
if err == io.EOF {
break
}
if err != nil {
close(kvs)
return count, err
}
count++
// We check for errors, if there are no errors we send value to channel.
select {
case kvs <- kv:
// OK
case <-ctx.Done():
x.TraceError(ctx, x.Errorf("Context timed out while streaming group: %v", group))
close(kvs)
return count, ctx.Err()
case err := <-che:
x.TraceError(ctx, x.Errorf("Error while doing a batch write for group: %v", group))
close(kvs)
return count, err
}
}
close(kvs)
if err := <-che; err != nil {
x.TraceError(ctx, x.Errorf("Error while doing a batch write for group: %v", group))
return count, err
}
x.Trace(ctx, "Streaming complete for group: %v", group)
return count, nil
}
func applyMutations(ctx context.Context, m *task.Mutations) error {
err := worker.MutateOverNetwork(ctx, m)
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while MutateOverNetwork"))
return err
}
return nil
}
// Get returns the value given a key for RocksDB.
func (s *Store) Get(key []byte) (*rdb.Slice, error) {
valSlice, err := s.db.Get(s.ropt, key)
if err != nil {
return nil, x.Wrapf(err, "Key: %v", key)
}
return valSlice, nil
}
func checkValidity(vm varMap) error {
for k, v := range vm {
typ := v.Type
if len(typ) == 0 {
return x.Errorf("Type of variable %v not specified", k)
}
// Ensure value is not nil if the variable is required.
if typ[len(typ)-1] == '!' {
if v.Value == "" {
return x.Errorf("Variable %v should be initialised", k)
}
typ = typ[:len(typ)-1]
}
// Type check the values.
if v.Value != "" {
switch typ {
case "int":
{
if _, err := strconv.ParseInt(v.Value, 0, 64); err != nil {
return x.Wrapf(err, "Expected an int but got %v", v.Value)
}
}
case "float":
{
if _, err := strconv.ParseFloat(v.Value, 64); err != nil {
return x.Wrapf(err, "Expected a float but got %v", v.Value)
}
}
case "bool":
{
if _, err := strconv.ParseBool(v.Value); err != nil {
return x.Wrapf(err, "Expected a bool but got %v", v.Value)
}
}
case "string": // Value is a valid string. No checks required.
default:
return x.Errorf("Type %v not supported", typ)
}
}
}
return nil
}
// AssignUidsOverNetwork assigns new uids and writes them to the umap.
func AssignUidsOverNetwork(ctx context.Context, umap map[string]uint64) error {
gid := group.BelongsTo("_uid_")
num := createNumQuery(gid, umap)
var ul *task.List
var err error
if groups().ServesGroup(gid) {
ul, err = assignUids(ctx, num)
if err != nil {
return err
}
} else {
_, addr := groups().Leader(gid)
p := pools().get(addr)
conn, err := p.Get()
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while retrieving connection"))
return err
}
defer p.Put(conn)
c := NewWorkerClient(conn)
ul, err = c.AssignUids(ctx, num)
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while getting uids"))
return err
}
}
x.AssertTruef(len(ul.Uids) == int(num.Val),
"Requested: %d != Retrieved Uids: %d", num.Val, len(ul.Uids))
i := 0
for k, v := range umap {
if v == 0 {
uid := ul.Uids[i]
umap[k] = uid // Write uids to map.
i++
}
}
return nil
}
// SortOverNetwork sends sort query over the network.
func SortOverNetwork(ctx context.Context, q *task.Sort) (*task.SortResult, error) {
gid := group.BelongsTo(q.Attr)
x.Trace(ctx, "worker.Sort attr: %v groupId: %v", q.Attr, gid)
if groups().ServesGroup(gid) {
// No need for a network call, as this should be run from within this instance.
return processSort(q)
}
// Send this over the network.
// TODO: Send the request to multiple servers as described in Jeff Dean's talk.
addr := groups().AnyServer(gid)
pl := pools().get(addr)
conn, err := pl.Get()
if err != nil {
return &emptySortResult, x.Wrapf(err, "SortOverNetwork: while retrieving connection.")
}
defer pl.Put(conn)
x.Trace(ctx, "Sending request to %v", addr)
c := NewWorkerClient(conn)
var reply *task.SortResult
cerr := make(chan error, 1)
go func() {
var err error
reply, err = c.Sort(ctx, q)
cerr <- err
}()
select {
case <-ctx.Done():
return &emptySortResult, ctx.Err()
case err := <-cerr:
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while calling Worker.Sort"))
}
return reply, err
}
}
func (w *Wal) Entries(gid uint32, fromTerm, fromIndex uint64) (es []raftpb.Entry, rerr error) {
start := w.entryKey(gid, fromTerm, fromIndex)
prefix := w.prefix(gid)
itr := w.wals.NewIterator()
defer itr.Close()
for itr.Seek(start); itr.ValidForPrefix(prefix); itr.Next() {
data := itr.Value().Data()
var e raftpb.Entry
if err := e.Unmarshal(data); err != nil {
return es, x.Wrapf(err, "While unmarshal raftpb.Entry")
}
es = append(es, e)
}
return
}
func (n *node) ProposeAndWait(ctx context.Context, proposal *task.Proposal) error {
if n.raft == nil {
return x.Errorf("RAFT isn't initialized yet")
}
proposal.Id = rand.Uint32()
slice := slicePool.Get().([]byte)
if len(slice) < proposal.Size() {
slice = make([]byte, proposal.Size())
}
defer slicePool.Put(slice)
upto, err := proposal.MarshalTo(slice)
if err != nil {
return err
}
proposalData := slice[:upto]
che := make(chan error, 1)
n.props.Store(proposal.Id, che)
if err = n.raft.Propose(ctx, proposalData); err != nil {
return x.Wrapf(err, "While proposing")
}
// Wait for the proposal to be committed.
if proposal.Mutations != nil {
x.Trace(ctx, "Waiting for the proposal: mutations.")
} else {
x.Trace(ctx, "Waiting for the proposal: membership update.")
}
select {
case err = <-che:
x.TraceError(ctx, err)
return err
case <-ctx.Done():
return ctx.Err()
}
}
// GetTokens returns the corresponding index keys based on the type
// of function.
func GetTokens(funcArgs []string) ([]string, *QueryData, error) {
x.AssertTruef(len(funcArgs) > 1, "Invalid function")
funcName := strings.ToLower(funcArgs[0])
switch funcName {
case "near":
if len(funcArgs) != 3 {
return nil, nil, x.Errorf("near function requires 3 arguments, but got %d",
len(funcArgs))
}
maxDist, err := strconv.ParseFloat(funcArgs[2], 64)
if err != nil {
return nil, nil, x.Wrapf(err, "Error while converting distance to float")
}
return queryTokens(QueryTypeNear, funcArgs[1], maxDist)
case "within":
if len(funcArgs) != 2 {
return nil, nil, x.Errorf("within function requires 2 arguments, but got %d",
len(funcArgs))
}
return queryTokens(QueryTypeWithin, funcArgs[1], 0.0)
case "contains":
if len(funcArgs) != 2 {
return nil, nil, x.Errorf("contains function requires 2 arguments, but got %d",
len(funcArgs))
}
return queryTokens(QueryTypeContains, funcArgs[1], 0.0)
case "intersects":
if len(funcArgs) != 2 {
return nil, nil, x.Errorf("intersects function requires 2 arguments, but got %d",
len(funcArgs))
}
return queryTokens(QueryTypeIntersects, funcArgs[1], 0.0)
default:
return nil, nil, x.Errorf("Invalid geo function")
}
}
// This method is used to execute the query and return the response to the
// client as a protocol buffer message.
func (s *grpcServer) Query(ctx context.Context,
req *graph.Request) (*graph.Response, error) {
var allocIds map[string]uint64
if rand.Float64() < *tracing {
tr := trace.New("Dgraph", "GrpcQuery")
defer tr.Finish()
ctx = trace.NewContext(ctx, tr)
}
resp := new(graph.Response)
if len(req.Query) == 0 && req.Mutation == nil {
x.TraceError(ctx, x.Errorf("Empty query and mutation."))
return resp, fmt.Errorf("Empty query and mutation.")
}
var l query.Latency
l.Start = time.Now()
x.Trace(ctx, "Query received: %v", req.Query)
gq, mu, err := gql.Parse(req.Query)
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while parsing query"))
return resp, err
}
// If mutations are part of the query, we run them through the mutation handler
// same as the http client.
if mu != nil && (len(mu.Set) > 0 || len(mu.Del) > 0) {
if allocIds, err = mutationHandler(ctx, mu); err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while handling mutations"))
return resp, err
}
}
// If mutations are sent as part of the mutation object in the request we run
// them here.
if req.Mutation != nil && (len(req.Mutation.Set) > 0 || len(req.Mutation.Del) > 0) {
if allocIds, err = runMutations(ctx, req.Mutation); err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while handling mutations"))
return resp, err
}
}
resp.AssignedUids = allocIds
if gq == nil || (gq.UID == 0 && len(gq.XID) == 0) {
return resp, err
}
sg, err := query.ToSubGraph(ctx, gq)
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while conversion to internal format"))
return resp, err
}
l.Parsing = time.Since(l.Start)
x.Trace(ctx, "Query parsed")
rch := make(chan error)
go query.ProcessGraph(ctx, sg, nil, rch)
err = <-rch
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while executing query"))
return resp, err
}
l.Processing = time.Since(l.Start) - l.Parsing
x.Trace(ctx, "Graph processed")
node, err := sg.ToProtocolBuffer(&l)
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while converting to ProtocolBuffer"))
return resp, err
}
resp.N = node
gl := new(graph.Latency)
gl.Parsing, gl.Processing, gl.Pb = l.Parsing.String(), l.Processing.String(),
l.ProtocolBuffer.String()
resp.L = gl
return resp, err
}
func queryHandler(w http.ResponseWriter, r *http.Request) {
// Add a limit on how many pending queries can be run in the system.
pendingQueries <- struct{}{}
defer func() { <-pendingQueries }()
addCorsHeaders(w)
if r.Method == "OPTIONS" {
return
}
if r.Method != "POST" {
x.SetStatus(w, x.ErrorInvalidMethod, "Invalid method")
return
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
if rand.Float64() < *tracing {
tr := trace.New("Dgraph", "Query")
defer tr.Finish()
ctx = trace.NewContext(ctx, tr)
}
var l query.Latency
l.Start = time.Now()
defer r.Body.Close()
req, err := ioutil.ReadAll(r.Body)
q := string(req)
if err != nil || len(q) == 0 {
x.TraceError(ctx, x.Wrapf(err, "Error while reading query"))
x.SetStatus(w, x.ErrorInvalidRequest, "Invalid request encountered.")
return
}
x.Trace(ctx, "Query received: %v", q)
gq, mu, err := gql.Parse(q)
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while parsing query"))
x.SetStatus(w, x.ErrorInvalidRequest, err.Error())
return
}
var allocIds map[string]uint64
var allocIdsStr map[string]string
// If we have mutations, run them first.
if mu != nil && (len(mu.Set) > 0 || len(mu.Del) > 0) {
if allocIds, err = mutationHandler(ctx, mu); err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while handling mutations"))
x.SetStatus(w, x.Error, err.Error())
return
}
// convert the new UIDs to hex string.
allocIdsStr = make(map[string]string)
for k, v := range allocIds {
allocIdsStr[k] = fmt.Sprintf("%#x", v)
}
}
if gq == nil || (gq.UID == 0 && gq.Func == nil && len(gq.XID) == 0) {
mp := map[string]interface{}{
"code": x.ErrorOk,
"message": "Done",
"uids": allocIdsStr,
}
if js, err := json.Marshal(mp); err == nil {
w.Write(js)
} else {
x.SetStatus(w, "Error", "Unable to marshal map")
}
return
}
sg, err := query.ToSubGraph(ctx, gq)
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while conversion o internal format"))
x.SetStatus(w, x.ErrorInvalidRequest, err.Error())
return
}
l.Parsing = time.Since(l.Start)
x.Trace(ctx, "Query parsed")
rch := make(chan error)
go query.ProcessGraph(ctx, sg, nil, rch)
err = <-rch
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while executing query"))
x.SetStatus(w, x.Error, err.Error())
return
}
l.Processing = time.Since(l.Start) - l.Parsing
x.Trace(ctx, "Graph processed")
if len(*dumpSubgraph) > 0 {
x.Checkf(os.MkdirAll(*dumpSubgraph, 0700), *dumpSubgraph)
s := time.Now().Format("20060102.150405.000000.gob")
filename := path.Join(*dumpSubgraph, s)
f, err := os.Create(filename)
x.Checkf(err, filename)
enc := gob.NewEncoder(f)
x.Check(enc.Encode(sg))
x.Checkf(f.Close(), filename)
}
js, err := sg.ToJSON(&l)
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while converting to Json"))
x.SetStatus(w, x.Error, err.Error())
return
}
x.Trace(ctx, "Latencies: Total: %v Parsing: %v Process: %v Json: %v",
time.Since(l.Start), l.Parsing, l.Processing, l.Json)
w.Header().Set("Content-Type", "application/json")
w.Write(js)
}
// ProcessGraph processes the SubGraph instance accumulating result for the query
// from different instances. Note: taskQuery is nil for root node.
func ProcessGraph(ctx context.Context, sg, parent *SubGraph, rch chan error) {
var err error
if len(sg.Attr) == 0 {
// If we have a filter SubGraph which only contains an operator,
// it won't have any attribute to work on.
// This is to allow providing SrcUIDs to the filter children.
sg.DestUIDs = sg.SrcUIDs
} else if parent == nil && len(sg.SrcFunc) == 0 {
// I am root. I don't have any function to execute, and my
// result has been prepared for me already.
sg.DestUIDs = algo.MergeSorted(sg.uidMatrix) // Could also be = sg.SrcUIDs
} else {
taskQuery := createTaskQuery(sg)
result, err := worker.ProcessTaskOverNetwork(ctx, taskQuery)
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while processing task"))
rch <- err
return
}
sg.uidMatrix = result.UidMatrix
sg.values = result.Values
if len(sg.values) > 0 {
v := sg.values[0]
x.Trace(ctx, "Sample value for attr: %v Val: %v", sg.Attr, string(v.Val))
}
sg.counts = result.Counts
if sg.Params.DoCount && len(sg.Filters) == 0 {
// If there is a filter, we need to do more work to get the actual count.
x.Trace(ctx, "Zero uids. Only count requested")
rch <- nil
return
}
if result.IntersectDest {
sg.DestUIDs = algo.IntersectSorted(result.UidMatrix)
} else {
sg.DestUIDs = algo.MergeSorted(result.UidMatrix)
}
}
if len(sg.DestUIDs.Uids) == 0 {
// Looks like we're done here. Be careful with nil srcUIDs!
x.Trace(ctx, "Zero uids for %q. Num attr children: %v", sg.Attr, len(sg.Children))
rch <- nil
return
}
// Apply filters if any.
if len(sg.Filters) > 0 {
// Run all filters in parallel.
filterChan := make(chan error, len(sg.Filters))
for _, filter := range sg.Filters {
filter.SrcUIDs = sg.DestUIDs
go ProcessGraph(ctx, filter, sg, filterChan)
}
for _ = range sg.Filters {
select {
case err = <-filterChan:
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while processing filter task"))
rch <- err
return
}
case <-ctx.Done():
x.TraceError(ctx, x.Wrapf(ctx.Err(), "Context done before full execution"))
rch <- ctx.Err()
return
}
}
// Now apply the results from filter.
var lists []*task.List
for _, filter := range sg.Filters {
lists = append(lists, filter.DestUIDs)
}
if sg.FilterOp == "|" {
sg.DestUIDs = algo.MergeSorted(lists)
} else {
sg.DestUIDs = algo.IntersectSorted(lists)
}
}
if len(sg.Params.Order) == 0 {
// There is no ordering. Just apply pagination and return.
if err = sg.applyPagination(ctx); err != nil {
rch <- err
return
}
} else {
// We need to sort first before pagination.
if err = sg.applyOrderAndPagination(ctx); err != nil {
rch <- err
return
}
}
// Here we consider handling _count_ with filtering. We do this after
// pagination because otherwise, we need to do the count with pagination
// taken into account. For example, a PL might have only 50 entries but the
// user wants to skip 100 entries and return 10 entries. In this case, you
// should return a count of 0, not 10.
if sg.Params.DoCount {
x.AssertTrue(len(sg.Filters) > 0)
sg.counts = make([]uint32, len(sg.uidMatrix))
for i, ul := range sg.uidMatrix {
// A possible optimization is to return the size of the intersection
// without forming the intersection.
algo.IntersectWith(ul, sg.DestUIDs)
sg.counts[i] = uint32(len(ul.Uids))
}
rch <- nil
return
}
childChan := make(chan error, len(sg.Children))
for i := 0; i < len(sg.Children); i++ {
child := sg.Children[i]
child.SrcUIDs = sg.DestUIDs // Make the connection.
go ProcessGraph(ctx, child, sg, childChan)
}
// Now get all the results back.
for _ = range sg.Children {
select {
case err = <-childChan:
if err != nil {
x.TraceError(ctx, x.Wrapf(err, "Error while processing child task"))
rch <- err
return
}
case <-ctx.Done():
x.TraceError(ctx, x.Wrapf(ctx.Err(), "Context done before full execution"))
rch <- ctx.Err()
return
}
}
rch <- nil
}
// queryTokens returns the tokens to be used to look up the geo index for a given filter.
func queryTokens(qt QueryType, data string, maxDistance float64) ([]string, *QueryData, error) {
// Try to parse the data as geo type.
var g types.Geo
geoData := strings.Replace(data, "'", "\"", -1)
err := g.UnmarshalText([]byte(geoData))
if err != nil {
return nil, nil, x.Wrapf(err, "Cannot decode given geoJson input")
}
var l *s2.Loop
var pt *s2.Point
switch v := g.T.(type) {
case *geom.Point:
p := pointFromPoint(v)
pt = &p
case *geom.Polygon:
l, err = loopFromPolygon(v)
if err != nil {
return nil, nil, err
}
default:
return nil, nil, x.Errorf("Cannot query using a geometry of type %T", v)
}
x.AssertTruef(l != nil || pt != nil, "We should have a point or a loop.")
parents, cover, err := indexCells(g)
if err != nil {
return nil, nil, err
}
switch qt {
case QueryTypeWithin:
// For a within query we only need to look at the objects whose parents match our cover.
// So we take our cover and prefix with the parentPrefix to look in the index.
toks := toTokens(cover, parentPrefix)
return toks, &QueryData{pt: pt, loop: l, qtype: qt}, nil
case QueryTypeContains:
if l != nil {
return nil, nil, x.Errorf("Cannot use a polygon in a contains query")
}
// For a contains query, we only need to look at the objects whose cover matches our
// parents. So we take our parents and prefix with the coverPrefix to look in the index.
return toTokens(parents, coverPrefix), &QueryData{pt: pt, qtype: qt}, nil
case QueryTypeNear:
if l != nil {
return nil, nil, x.Errorf("Cannot use a polygon in a near query")
}
return nearQueryKeys(*pt, maxDistance)
case QueryTypeIntersects:
// An intersects query is essentially the union of contains and within. So we look at all
// the objects whose parents match our cover as well as all the objects whose cover matches
// our parents.
toks := parentCoverTokens(parents, cover)
return toks, &QueryData{pt: pt, loop: l, qtype: qt}, nil
default:
return nil, nil, x.Errorf("Unknown query type")
}
}
