// ProcessTaskOverNetwork is used to process the query and get the result from // the instance which stores posting list corresponding to the predicate in the // query. func ProcessTaskOverNetwork(ctx context.Context, q *task.Query) (*task.Result, error) { attr := q.Attr gid := group.BelongsTo(attr) x.Trace(ctx, "attr: %v groupId: %v", attr, gid) if groups().ServesGroup(gid) { // No need for a network call, as this should be run from within this instance. return processTask(q) } // Send this over the network. // TODO: Send the request to multiple servers as described in Jeff Dean's talk. addr := groups().AnyServer(gid) pl := pools().get(addr) conn, err := pl.Get() if err != nil { return &emptyResult, x.Wrapf(err, "ProcessTaskOverNetwork: while retrieving connection.") } defer pl.Put(conn) x.Trace(ctx, "Sending request to %v", addr) c := NewWorkerClient(conn) reply, err := c.ServeTask(ctx, q) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while calling Worker.ServeTask")) return &emptyResult, err } x.Trace(ctx, "Reply from server. length: %v Addr: %v Attr: %v", len(reply.UidMatrix), addr, attr) return reply, nil }
func addIndexMutation(ctx context.Context, attr, token string, tokensTable *TokensTable, edge *task.DirectedEdge, del bool) { key := x.IndexKey(attr, token) plist, decr := GetOrCreate(key) defer decr() x.AssertTruef(plist != nil, "plist is nil [%s] %d %s", token, edge.ValueId, edge.Attr) if del { _, err := plist.AddMutation(ctx, edge, Del) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error deleting %s for attr %s entity %d: %v", token, edge.Attr, edge.Entity)) } indexLog.Printf("DEL [%s] [%d] OldTerm [%s]", edge.Attr, edge.Entity, token) } else { _, err := plist.AddMutation(ctx, edge, Set) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error adding %s for attr %s entity %d: %v", token, edge.Attr, edge.Entity)) } indexLog.Printf("SET [%s] [%d] NewTerm [%s]", edge.Attr, edge.Entity, token) tokensTable.Add(token) } }
func (w *Wal) HardState(gid uint32) (hd raftpb.HardState, rerr error) { slice, err := w.wals.Get(w.hardStateKey(gid)) if err != nil || slice == nil { return hd, x.Wrapf(err, "While getting hardstate") } rerr = x.Wrapf(hd.Unmarshal(slice.Data()), "While unmarshal hardstate") slice.Free() return }
func (w *Wal) Snapshot(gid uint32) (snap raftpb.Snapshot, rerr error) { slice, err := w.wals.Get(w.snapshotKey(gid)) if err != nil || slice == nil { return snap, x.Wrapf(err, "While getting snapshot") } rerr = x.Wrapf(snap.Unmarshal(slice.Data()), "While unmarshal snapshot") slice.Free() return }
func convertToEdges(ctx context.Context, nquads []rdf.NQuad) (mutationResult, error) { var edges []*task.DirectedEdge var mr mutationResult newUids := make(map[string]uint64) for _, nq := range nquads { if strings.HasPrefix(nq.Subject, "_new_:") { newUids[nq.Subject] = 0 } else if !strings.HasPrefix(nq.Subject, "_uid_:") { uid, err := rdf.GetUid(nq.Subject) x.Check(err) newUids[nq.Subject] = uid } if len(nq.ObjectId) > 0 { if strings.HasPrefix(nq.ObjectId, "_new_:") { newUids[nq.ObjectId] = 0 } else if !strings.HasPrefix(nq.ObjectId, "_uid_:") { uid, err := rdf.GetUid(nq.ObjectId) x.Check(err) newUids[nq.ObjectId] = uid } } } if len(newUids) > 0 { if err := worker.AssignUidsOverNetwork(ctx, newUids); err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while GetOrAssignUidsOverNetwork")) return mr, err } } for _, nq := range nquads { // Get edges from nquad using newUids. edge, err := nq.ToEdgeUsing(newUids) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while converting to edge: %v", nq)) return mr, err } edges = append(edges, edge) } resultUids := make(map[string]uint64) // Strip out _new_: prefix from the keys. for k, v := range newUids { if strings.HasPrefix(k, "_new_:") { resultUids[k[6:]] = v } } mr = mutationResult{ edges: edges, newUids: resultUids, } return mr, nil }
// Store stores the snapshot, hardstate and entries for a given RAFT group. func (w *Wal) Store(gid uint32, s raftpb.Snapshot, h raftpb.HardState, es []raftpb.Entry) error { b := w.wals.NewWriteBatch() defer b.Destroy() if !raft.IsEmptySnap(s) { data, err := s.Marshal() if err != nil { return x.Wrapf(err, "wal.Store: While marshal snapshot") } b.Put(w.snapshotKey(gid), data) } if !raft.IsEmptyHardState(h) { data, err := h.Marshal() if err != nil { return x.Wrapf(err, "wal.Store: While marshal hardstate") } b.Put(w.hardStateKey(gid), data) } var t, i uint64 for _, e := range es { t, i = e.Term, e.Index data, err := e.Marshal() if err != nil { return x.Wrapf(err, "wal.Store: While marshal entry") } k := w.entryKey(gid, e.Term, e.Index) b.Put(k, data) } // If we get no entries, then the default value of t and i would be zero. That would // end up deleting all the previous valid raft entry logs. This check avoids that. if t > 0 || i > 0 { // Delete all keys above this index. start := w.entryKey(gid, t, i+1) prefix := w.prefix(gid) itr := w.wals.NewIterator() defer itr.Close() for itr.Seek(start); itr.ValidForPrefix(prefix); itr.Next() { b.Delete(itr.Key().Data()) } } err := w.wals.WriteBatch(b) return x.Wrapf(err, "wal.Store: While WriteBatch") }
// MutateOverNetwork checks which group should be running the mutations // according to fingerprint of the predicate and sends it to that instance. func MutateOverNetwork(ctx context.Context, m *task.Mutations) error { mutationMap := make(map[uint32]*task.Mutations) addToMutationMap(mutationMap, m.Set, set) addToMutationMap(mutationMap, m.Del, del) errors := make(chan error, len(mutationMap)) for gid, mu := range mutationMap { proposeOrSend(ctx, gid, mu, errors) } // Wait for all the goroutines to reply back. // We return if an error was returned or the parent called ctx.Done() for i := 0; i < len(mutationMap); i++ { select { case err := <-errors: if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while running all mutations")) return err } case <-ctx.Done(): return ctx.Err() } } close(errors) return nil }
func convertToNQuad(ctx context.Context, mutation string) ([]rdf.NQuad, error) { var nquads []rdf.NQuad r := strings.NewReader(mutation) reader := bufio.NewReader(r) x.Trace(ctx, "Converting to NQuad") var strBuf bytes.Buffer var err error for { err = x.ReadLine(reader, &strBuf) if err != nil { break } ln := strings.Trim(strBuf.String(), " \t") if len(ln) == 0 { continue } nq, err := rdf.Parse(ln) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while parsing RDF")) return nquads, err } nquads = append(nquads, nq) } if err != io.EOF { return nquads, err } return nquads, nil }
// PopulateShard gets data for predicate pred from server with id serverId and // writes it to RocksDB. func populateShard(ctx context.Context, pl *pool, group uint32) (int, error) { gkeys, err := generateGroup(group) if err != nil { return 0, x.Wrapf(err, "While generating keys group") } conn, err := pl.Get() if err != nil { return 0, err } defer pl.Put(conn) c := NewWorkerClient(conn) stream, err := c.PredicateData(context.Background(), gkeys) if err != nil { return 0, err } x.Trace(ctx, "Streaming data for group: %v", group) kvs := make(chan *task.KV, 1000) che := make(chan error) go writeBatch(ctx, kvs, che) // We can use count to check the number of posting lists returned in tests. count := 0 for { kv, err := stream.Recv() if err == io.EOF { break } if err != nil { close(kvs) return count, err } count++ // We check for errors, if there are no errors we send value to channel. select { case kvs <- kv: // OK case <-ctx.Done(): x.TraceError(ctx, x.Errorf("Context timed out while streaming group: %v", group)) close(kvs) return count, ctx.Err() case err := <-che: x.TraceError(ctx, x.Errorf("Error while doing a batch write for group: %v", group)) close(kvs) return count, err } } close(kvs) if err := <-che; err != nil { x.TraceError(ctx, x.Errorf("Error while doing a batch write for group: %v", group)) return count, err } x.Trace(ctx, "Streaming complete for group: %v", group) return count, nil }
func applyMutations(ctx context.Context, m *task.Mutations) error { err := worker.MutateOverNetwork(ctx, m) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while MutateOverNetwork")) return err } return nil }
// Get returns the value given a key for RocksDB. func (s *Store) Get(key []byte) (*rdb.Slice, error) { valSlice, err := s.db.Get(s.ropt, key) if err != nil { return nil, x.Wrapf(err, "Key: %v", key) } return valSlice, nil }
func checkValidity(vm varMap) error { for k, v := range vm { typ := v.Type if len(typ) == 0 { return x.Errorf("Type of variable %v not specified", k) } // Ensure value is not nil if the variable is required. if typ[len(typ)-1] == '!' { if v.Value == "" { return x.Errorf("Variable %v should be initialised", k) } typ = typ[:len(typ)-1] } // Type check the values. if v.Value != "" { switch typ { case "int": { if _, err := strconv.ParseInt(v.Value, 0, 64); err != nil { return x.Wrapf(err, "Expected an int but got %v", v.Value) } } case "float": { if _, err := strconv.ParseFloat(v.Value, 64); err != nil { return x.Wrapf(err, "Expected a float but got %v", v.Value) } } case "bool": { if _, err := strconv.ParseBool(v.Value); err != nil { return x.Wrapf(err, "Expected a bool but got %v", v.Value) } } case "string": // Value is a valid string. No checks required. default: return x.Errorf("Type %v not supported", typ) } } } return nil }
// AssignUidsOverNetwork assigns new uids and writes them to the umap. func AssignUidsOverNetwork(ctx context.Context, umap map[string]uint64) error { gid := group.BelongsTo("_uid_") num := createNumQuery(gid, umap) var ul *task.List var err error if groups().ServesGroup(gid) { ul, err = assignUids(ctx, num) if err != nil { return err } } else { _, addr := groups().Leader(gid) p := pools().get(addr) conn, err := p.Get() if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while retrieving connection")) return err } defer p.Put(conn) c := NewWorkerClient(conn) ul, err = c.AssignUids(ctx, num) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while getting uids")) return err } } x.AssertTruef(len(ul.Uids) == int(num.Val), "Requested: %d != Retrieved Uids: %d", num.Val, len(ul.Uids)) i := 0 for k, v := range umap { if v == 0 { uid := ul.Uids[i] umap[k] = uid // Write uids to map. i++ } } return nil }
// SortOverNetwork sends sort query over the network. func SortOverNetwork(ctx context.Context, q *task.Sort) (*task.SortResult, error) { gid := group.BelongsTo(q.Attr) x.Trace(ctx, "worker.Sort attr: %v groupId: %v", q.Attr, gid) if groups().ServesGroup(gid) { // No need for a network call, as this should be run from within this instance. return processSort(q) } // Send this over the network. // TODO: Send the request to multiple servers as described in Jeff Dean's talk. addr := groups().AnyServer(gid) pl := pools().get(addr) conn, err := pl.Get() if err != nil { return &emptySortResult, x.Wrapf(err, "SortOverNetwork: while retrieving connection.") } defer pl.Put(conn) x.Trace(ctx, "Sending request to %v", addr) c := NewWorkerClient(conn) var reply *task.SortResult cerr := make(chan error, 1) go func() { var err error reply, err = c.Sort(ctx, q) cerr <- err }() select { case <-ctx.Done(): return &emptySortResult, ctx.Err() case err := <-cerr: if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while calling Worker.Sort")) } return reply, err } }
func (w *Wal) Entries(gid uint32, fromTerm, fromIndex uint64) (es []raftpb.Entry, rerr error) { start := w.entryKey(gid, fromTerm, fromIndex) prefix := w.prefix(gid) itr := w.wals.NewIterator() defer itr.Close() for itr.Seek(start); itr.ValidForPrefix(prefix); itr.Next() { data := itr.Value().Data() var e raftpb.Entry if err := e.Unmarshal(data); err != nil { return es, x.Wrapf(err, "While unmarshal raftpb.Entry") } es = append(es, e) } return }
func (n *node) ProposeAndWait(ctx context.Context, proposal *task.Proposal) error { if n.raft == nil { return x.Errorf("RAFT isn't initialized yet") } proposal.Id = rand.Uint32() slice := slicePool.Get().([]byte) if len(slice) < proposal.Size() { slice = make([]byte, proposal.Size()) } defer slicePool.Put(slice) upto, err := proposal.MarshalTo(slice) if err != nil { return err } proposalData := slice[:upto] che := make(chan error, 1) n.props.Store(proposal.Id, che) if err = n.raft.Propose(ctx, proposalData); err != nil { return x.Wrapf(err, "While proposing") } // Wait for the proposal to be committed. if proposal.Mutations != nil { x.Trace(ctx, "Waiting for the proposal: mutations.") } else { x.Trace(ctx, "Waiting for the proposal: membership update.") } select { case err = <-che: x.TraceError(ctx, err) return err case <-ctx.Done(): return ctx.Err() } }
// GetTokens returns the corresponding index keys based on the type // of function. func GetTokens(funcArgs []string) ([]string, *QueryData, error) { x.AssertTruef(len(funcArgs) > 1, "Invalid function") funcName := strings.ToLower(funcArgs[0]) switch funcName { case "near": if len(funcArgs) != 3 { return nil, nil, x.Errorf("near function requires 3 arguments, but got %d", len(funcArgs)) } maxDist, err := strconv.ParseFloat(funcArgs[2], 64) if err != nil { return nil, nil, x.Wrapf(err, "Error while converting distance to float") } return queryTokens(QueryTypeNear, funcArgs[1], maxDist) case "within": if len(funcArgs) != 2 { return nil, nil, x.Errorf("within function requires 2 arguments, but got %d", len(funcArgs)) } return queryTokens(QueryTypeWithin, funcArgs[1], 0.0) case "contains": if len(funcArgs) != 2 { return nil, nil, x.Errorf("contains function requires 2 arguments, but got %d", len(funcArgs)) } return queryTokens(QueryTypeContains, funcArgs[1], 0.0) case "intersects": if len(funcArgs) != 2 { return nil, nil, x.Errorf("intersects function requires 2 arguments, but got %d", len(funcArgs)) } return queryTokens(QueryTypeIntersects, funcArgs[1], 0.0) default: return nil, nil, x.Errorf("Invalid geo function") } }
// This method is used to execute the query and return the response to the // client as a protocol buffer message. func (s *grpcServer) Query(ctx context.Context, req *graph.Request) (*graph.Response, error) { var allocIds map[string]uint64 if rand.Float64() < *tracing { tr := trace.New("Dgraph", "GrpcQuery") defer tr.Finish() ctx = trace.NewContext(ctx, tr) } resp := new(graph.Response) if len(req.Query) == 0 && req.Mutation == nil { x.TraceError(ctx, x.Errorf("Empty query and mutation.")) return resp, fmt.Errorf("Empty query and mutation.") } var l query.Latency l.Start = time.Now() x.Trace(ctx, "Query received: %v", req.Query) gq, mu, err := gql.Parse(req.Query) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while parsing query")) return resp, err } // If mutations are part of the query, we run them through the mutation handler // same as the http client. if mu != nil && (len(mu.Set) > 0 || len(mu.Del) > 0) { if allocIds, err = mutationHandler(ctx, mu); err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while handling mutations")) return resp, err } } // If mutations are sent as part of the mutation object in the request we run // them here. if req.Mutation != nil && (len(req.Mutation.Set) > 0 || len(req.Mutation.Del) > 0) { if allocIds, err = runMutations(ctx, req.Mutation); err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while handling mutations")) return resp, err } } resp.AssignedUids = allocIds if gq == nil || (gq.UID == 0 && len(gq.XID) == 0) { return resp, err } sg, err := query.ToSubGraph(ctx, gq) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while conversion to internal format")) return resp, err } l.Parsing = time.Since(l.Start) x.Trace(ctx, "Query parsed") rch := make(chan error) go query.ProcessGraph(ctx, sg, nil, rch) err = <-rch if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while executing query")) return resp, err } l.Processing = time.Since(l.Start) - l.Parsing x.Trace(ctx, "Graph processed") node, err := sg.ToProtocolBuffer(&l) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while converting to ProtocolBuffer")) return resp, err } resp.N = node gl := new(graph.Latency) gl.Parsing, gl.Processing, gl.Pb = l.Parsing.String(), l.Processing.String(), l.ProtocolBuffer.String() resp.L = gl return resp, err }
func queryHandler(w http.ResponseWriter, r *http.Request) { // Add a limit on how many pending queries can be run in the system. pendingQueries <- struct{}{} defer func() { <-pendingQueries }() addCorsHeaders(w) if r.Method == "OPTIONS" { return } if r.Method != "POST" { x.SetStatus(w, x.ErrorInvalidMethod, "Invalid method") return } ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() if rand.Float64() < *tracing { tr := trace.New("Dgraph", "Query") defer tr.Finish() ctx = trace.NewContext(ctx, tr) } var l query.Latency l.Start = time.Now() defer r.Body.Close() req, err := ioutil.ReadAll(r.Body) q := string(req) if err != nil || len(q) == 0 { x.TraceError(ctx, x.Wrapf(err, "Error while reading query")) x.SetStatus(w, x.ErrorInvalidRequest, "Invalid request encountered.") return } x.Trace(ctx, "Query received: %v", q) gq, mu, err := gql.Parse(q) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while parsing query")) x.SetStatus(w, x.ErrorInvalidRequest, err.Error()) return } var allocIds map[string]uint64 var allocIdsStr map[string]string // If we have mutations, run them first. if mu != nil && (len(mu.Set) > 0 || len(mu.Del) > 0) { if allocIds, err = mutationHandler(ctx, mu); err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while handling mutations")) x.SetStatus(w, x.Error, err.Error()) return } // convert the new UIDs to hex string. allocIdsStr = make(map[string]string) for k, v := range allocIds { allocIdsStr[k] = fmt.Sprintf("%#x", v) } } if gq == nil || (gq.UID == 0 && gq.Func == nil && len(gq.XID) == 0) { mp := map[string]interface{}{ "code": x.ErrorOk, "message": "Done", "uids": allocIdsStr, } if js, err := json.Marshal(mp); err == nil { w.Write(js) } else { x.SetStatus(w, "Error", "Unable to marshal map") } return } sg, err := query.ToSubGraph(ctx, gq) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while conversion o internal format")) x.SetStatus(w, x.ErrorInvalidRequest, err.Error()) return } l.Parsing = time.Since(l.Start) x.Trace(ctx, "Query parsed") rch := make(chan error) go query.ProcessGraph(ctx, sg, nil, rch) err = <-rch if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while executing query")) x.SetStatus(w, x.Error, err.Error()) return } l.Processing = time.Since(l.Start) - l.Parsing x.Trace(ctx, "Graph processed") if len(*dumpSubgraph) > 0 { x.Checkf(os.MkdirAll(*dumpSubgraph, 0700), *dumpSubgraph) s := time.Now().Format("20060102.150405.000000.gob") filename := path.Join(*dumpSubgraph, s) f, err := os.Create(filename) x.Checkf(err, filename) enc := gob.NewEncoder(f) x.Check(enc.Encode(sg)) x.Checkf(f.Close(), filename) } js, err := sg.ToJSON(&l) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while converting to Json")) x.SetStatus(w, x.Error, err.Error()) return } x.Trace(ctx, "Latencies: Total: %v Parsing: %v Process: %v Json: %v", time.Since(l.Start), l.Parsing, l.Processing, l.Json) w.Header().Set("Content-Type", "application/json") w.Write(js) }
// ProcessGraph processes the SubGraph instance accumulating result for the query // from different instances. Note: taskQuery is nil for root node. func ProcessGraph(ctx context.Context, sg, parent *SubGraph, rch chan error) { var err error if len(sg.Attr) == 0 { // If we have a filter SubGraph which only contains an operator, // it won't have any attribute to work on. // This is to allow providing SrcUIDs to the filter children. sg.DestUIDs = sg.SrcUIDs } else if parent == nil && len(sg.SrcFunc) == 0 { // I am root. I don't have any function to execute, and my // result has been prepared for me already. sg.DestUIDs = algo.MergeSorted(sg.uidMatrix) // Could also be = sg.SrcUIDs } else { taskQuery := createTaskQuery(sg) result, err := worker.ProcessTaskOverNetwork(ctx, taskQuery) if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while processing task")) rch <- err return } sg.uidMatrix = result.UidMatrix sg.values = result.Values if len(sg.values) > 0 { v := sg.values[0] x.Trace(ctx, "Sample value for attr: %v Val: %v", sg.Attr, string(v.Val)) } sg.counts = result.Counts if sg.Params.DoCount && len(sg.Filters) == 0 { // If there is a filter, we need to do more work to get the actual count. x.Trace(ctx, "Zero uids. Only count requested") rch <- nil return } if result.IntersectDest { sg.DestUIDs = algo.IntersectSorted(result.UidMatrix) } else { sg.DestUIDs = algo.MergeSorted(result.UidMatrix) } } if len(sg.DestUIDs.Uids) == 0 { // Looks like we're done here. Be careful with nil srcUIDs! x.Trace(ctx, "Zero uids for %q. Num attr children: %v", sg.Attr, len(sg.Children)) rch <- nil return } // Apply filters if any. if len(sg.Filters) > 0 { // Run all filters in parallel. filterChan := make(chan error, len(sg.Filters)) for _, filter := range sg.Filters { filter.SrcUIDs = sg.DestUIDs go ProcessGraph(ctx, filter, sg, filterChan) } for _ = range sg.Filters { select { case err = <-filterChan: if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while processing filter task")) rch <- err return } case <-ctx.Done(): x.TraceError(ctx, x.Wrapf(ctx.Err(), "Context done before full execution")) rch <- ctx.Err() return } } // Now apply the results from filter. var lists []*task.List for _, filter := range sg.Filters { lists = append(lists, filter.DestUIDs) } if sg.FilterOp == "|" { sg.DestUIDs = algo.MergeSorted(lists) } else { sg.DestUIDs = algo.IntersectSorted(lists) } } if len(sg.Params.Order) == 0 { // There is no ordering. Just apply pagination and return. if err = sg.applyPagination(ctx); err != nil { rch <- err return } } else { // We need to sort first before pagination. if err = sg.applyOrderAndPagination(ctx); err != nil { rch <- err return } } // Here we consider handling _count_ with filtering. We do this after // pagination because otherwise, we need to do the count with pagination // taken into account. For example, a PL might have only 50 entries but the // user wants to skip 100 entries and return 10 entries. In this case, you // should return a count of 0, not 10. if sg.Params.DoCount { x.AssertTrue(len(sg.Filters) > 0) sg.counts = make([]uint32, len(sg.uidMatrix)) for i, ul := range sg.uidMatrix { // A possible optimization is to return the size of the intersection // without forming the intersection. algo.IntersectWith(ul, sg.DestUIDs) sg.counts[i] = uint32(len(ul.Uids)) } rch <- nil return } childChan := make(chan error, len(sg.Children)) for i := 0; i < len(sg.Children); i++ { child := sg.Children[i] child.SrcUIDs = sg.DestUIDs // Make the connection. go ProcessGraph(ctx, child, sg, childChan) } // Now get all the results back. for _ = range sg.Children { select { case err = <-childChan: if err != nil { x.TraceError(ctx, x.Wrapf(err, "Error while processing child task")) rch <- err return } case <-ctx.Done(): x.TraceError(ctx, x.Wrapf(ctx.Err(), "Context done before full execution")) rch <- ctx.Err() return } } rch <- nil }
// queryTokens returns the tokens to be used to look up the geo index for a given filter. func queryTokens(qt QueryType, data string, maxDistance float64) ([]string, *QueryData, error) { // Try to parse the data as geo type. var g types.Geo geoData := strings.Replace(data, "'", "\"", -1) err := g.UnmarshalText([]byte(geoData)) if err != nil { return nil, nil, x.Wrapf(err, "Cannot decode given geoJson input") } var l *s2.Loop var pt *s2.Point switch v := g.T.(type) { case *geom.Point: p := pointFromPoint(v) pt = &p case *geom.Polygon: l, err = loopFromPolygon(v) if err != nil { return nil, nil, err } default: return nil, nil, x.Errorf("Cannot query using a geometry of type %T", v) } x.AssertTruef(l != nil || pt != nil, "We should have a point or a loop.") parents, cover, err := indexCells(g) if err != nil { return nil, nil, err } switch qt { case QueryTypeWithin: // For a within query we only need to look at the objects whose parents match our cover. // So we take our cover and prefix with the parentPrefix to look in the index. toks := toTokens(cover, parentPrefix) return toks, &QueryData{pt: pt, loop: l, qtype: qt}, nil case QueryTypeContains: if l != nil { return nil, nil, x.Errorf("Cannot use a polygon in a contains query") } // For a contains query, we only need to look at the objects whose cover matches our // parents. So we take our parents and prefix with the coverPrefix to look in the index. return toTokens(parents, coverPrefix), &QueryData{pt: pt, qtype: qt}, nil case QueryTypeNear: if l != nil { return nil, nil, x.Errorf("Cannot use a polygon in a near query") } return nearQueryKeys(*pt, maxDistance) case QueryTypeIntersects: // An intersects query is essentially the union of contains and within. So we look at all // the objects whose parents match our cover as well as all the objects whose cover matches // our parents. toks := parentCoverTokens(parents, cover) return toks, &QueryData{pt: pt, loop: l, qtype: qt}, nil default: return nil, nil, x.Errorf("Unknown query type") } }