Beispiel #1
0
func (n *node) processSnapshot(s raftpb.Snapshot) {
	lead := n.raft.Status().Lead
	if lead == 0 {
		return
	}
	addr := n.peers.Get(lead)
	x.AssertTruef(addr != "", "Should have the leader address: %v", lead)
	pool := pools().get(addr)
	x.AssertTruef(pool != nil, "Leader: %d pool should not be nil", lead)

	_, err := populateShard(context.TODO(), pool, 0)
	x.Checkf(err, "processSnapshot")
}
Beispiel #2
0
// returns true if the geometry represented by uid/attr intersects the given loop or point
func (q QueryData) intersects(g types.Geo) bool {
	x.AssertTruef(q.pt != nil || q.loop != nil, "At least a point or loop should be defined.")
	switch v := g.T.(type) {
	case *geom.Point:
		p := pointFromPoint(v)
		if q.pt != nil {
			// Points only intersect if they are the same. (We allow for small rounding errors)
			return q.pt.ApproxEqual(p)
		}
		// else loop is not nil
		return q.loop.ContainsPoint(p)

	case *geom.Polygon:
		l, err := loopFromPolygon(v)
		if err != nil {
			return false
		}
		if q.pt != nil {
			return l.ContainsPoint(*q.pt)
		}
		// else loop is not nil
		return Intersects(l, q.loop)
	default:
		// A type that we don't know how to handle.
		return false
	}
}
Beispiel #3
0
// FilterUids filters the uids based on the corresponding values and QueryData.
func FilterUids(uids *task.List, values []*task.Value, q *QueryData) *task.List {
	x.AssertTruef(len(values) == len(uids.Uids), "lengths not matching")
	rv := &task.List{}
	for i := 0; i < len(values); i++ {
		valBytes := values[i].Val
		if bytes.Equal(valBytes, nil) {
			continue
		}
		vType := values[i].ValType
		if types.TypeID(vType) != types.GeoID {
			continue
		}
		var g types.Geo
		if err := g.UnmarshalBinary(valBytes); err != nil {
			continue
		}

		if !q.MatchesFilter(g) {
			continue
		}

		// we matched the geo filter, add the uid to the list
		rv.Uids = append(rv.Uids, uids.Uids[i])
	}
	return rv
}
Beispiel #4
0
func (p *proposals) Store(pid uint32, ch chan error) {
	p.Lock()
	defer p.Unlock()
	_, has := p.ids[pid]
	x.AssertTruef(!has, "Same proposal is being stored again.")
	p.ids[pid] = ch
}
Beispiel #5
0
// ServeTask is used to respond to a query.
func (w *grpcWorker) ServeTask(ctx context.Context, q *task.Query) (*task.Result, error) {
	if ctx.Err() != nil {
		return &emptyResult, ctx.Err()
	}

	gid := group.BelongsTo(q.Attr)
	x.Trace(ctx, "Attribute: %q NumUids: %v groupId: %v ServeTask", q.Attr, len(q.Uids), gid)

	var reply *task.Result
	x.AssertTruef(groups().ServesGroup(gid),
		"attr: %q groupId: %v Request sent to wrong server.", q.Attr, gid)

	c := make(chan error, 1)
	go func() {
		var err error
		reply, err = processTask(q)
		c <- err
	}()

	select {
	case <-ctx.Done():
		return reply, ctx.Err()
	case err := <-c:
		return reply, err
	}
}
Beispiel #6
0
// Sort is used to sort given UID matrix.
func (w *grpcWorker) Sort(ctx context.Context, s *task.Sort) (*task.SortResult, error) {
	if ctx.Err() != nil {
		return &emptySortResult, ctx.Err()
	}

	gid := group.BelongsTo(s.Attr)
	//x.Trace(ctx, "Attribute: %q NumUids: %v groupId: %v Sort", q.Attr(), q.UidsLength(), gid)

	var reply *task.SortResult
	x.AssertTruef(groups().ServesGroup(gid),
		"attr: %q groupId: %v Request sent to wrong server.", s.Attr, gid)

	c := make(chan error, 1)
	go func() {
		var err error
		reply, err = processSort(s)
		c <- err
	}()

	select {
	case <-ctx.Done():
		return &emptySortResult, ctx.Err()
	case err := <-c:
		return reply, err
	}
}
Beispiel #7
0
func addIndexMutation(ctx context.Context, attr, token string,
	tokensTable *TokensTable, edge *task.DirectedEdge, del bool) {
	key := x.IndexKey(attr, token)
	plist, decr := GetOrCreate(key)
	defer decr()

	x.AssertTruef(plist != nil, "plist is nil [%s] %d %s",
		token, edge.ValueId, edge.Attr)
	if del {
		_, err := plist.AddMutation(ctx, edge, Del)
		if err != nil {
			x.TraceError(ctx, x.Wrapf(err,
				"Error deleting %s for attr %s entity %d: %v",
				token, edge.Attr, edge.Entity))
		}
		indexLog.Printf("DEL [%s] [%d] OldTerm [%s]",
			edge.Attr, edge.Entity, token)

	} else {
		_, err := plist.AddMutation(ctx, edge, Set)
		if err != nil {
			x.TraceError(ctx, x.Wrapf(err,
				"Error adding %s for attr %s entity %d: %v",
				token, edge.Attr, edge.Entity))
		}
		indexLog.Printf("SET [%s] [%d] NewTerm [%s]",
			edge.Attr, edge.Entity, token)

		tokensTable.Add(token)
	}
}
Beispiel #8
0
func (l *List) decr() {
	val := atomic.AddInt32(&l.refcount, -1)
	x.AssertTruef(val >= 0, "List reference should never be less than zero: %v", val)
	if val > 0 {
		return
	}
	listPool.Put(l)
}
Beispiel #9
0
func getTokens(funcArgs []string) ([]string, error) {
	x.AssertTruef(len(funcArgs) > 1, "Invalid function")
	if len(funcArgs) != 2 {
		return nil, x.Errorf("Function requires 2 arguments, but got %d",
			len(funcArgs))
	}
	return getStringTokens(funcArgs[1])
}
Beispiel #10
0
func parsePeer(peer string) (uint64, string) {
	x.AssertTrue(len(peer) > 0)

	kv := strings.SplitN(peer, ":", 2)
	x.AssertTruef(len(kv) == 2, "Invalid peer format: %v", peer)
	pid, err := strconv.ParseUint(kv[0], 10, 64)
	x.Checkf(err, "Invalid peer id: %v", kv[0])
	// TODO: Validate the url kv[1]
	return pid, kv[1]
}
Beispiel #11
0
func parentCoverTokens(parents s2.CellUnion, cover s2.CellUnion) []string {
	// We index parents and cover using different prefix, that makes it more
	// performant at query time to only look up parents/cover depending on what
	// kind of query it is.
	tokens := make([]string, 0, len(parents)+len(cover))
	tokens = appendTokens(tokens, parents, parentPrefix)
	tokens = appendTokens(tokens, cover, coverPrefix)
	x.AssertTruef(len(tokens) == len(parents)+len(cover), "%d %d %d",
		len(tokens), len(parents), len(cover))
	return tokens
}
Beispiel #12
0
func newPosting(t *task.DirectedEdge, op uint32) *types.Posting {
	x.AssertTruef(bytes.Equal(t.Value, nil) || t.ValueId == math.MaxUint64,
		"This should have been set by the caller.")

	return &types.Posting{
		Uid:     t.ValueId,
		Value:   t.Value,
		ValType: uint32(t.ValueType),
		Label:   t.Label,
		Op:      op,
	}
}
Beispiel #13
0
func (n *node) send(m raftpb.Message) {
	x.AssertTruef(n.id != m.To, "Seding message to itself")
	data, err := m.Marshal()
	x.Check(err)
	if m.Type != raftpb.MsgHeartbeat && m.Type != raftpb.MsgHeartbeatResp {
		fmt.Printf("\t\tSENDING: %v %v-->%v\n", m.Type, m.From, m.To)
	}
	select {
	case n.messages <- sendmsg{to: m.To, data: data}:
		// pass
	default:
		log.Fatalf("Unable to push messages to channel in send")
	}
}
Beispiel #14
0
func (g *groupi) newNode(groupId uint32, nodeId uint64, publicAddr string) *node {
	g.Lock()
	defer g.Unlock()
	if g.local == nil {
		g.local = make(map[uint32]*node)
	}

	node := newNode(groupId, nodeId, publicAddr)
	if _, has := g.local[groupId]; has {
		x.AssertTruef(false, "Didn't expect a node in RAFT group mapping: %v", groupId)
	}
	g.local[groupId] = node
	return node
}
Beispiel #15
0
// processSort does either a coarse or a fine sort.
func processSort(ts *task.Sort) (*task.SortResult, error) {
	attr := ts.Attr
	x.AssertTruef(ts.Count > 0,
		("We do not yet support negative or infinite count with sorting: %s %d. " +
			"Try flipping order and return first few elements instead."),
		attr, ts.Count)

	n := len(ts.UidMatrix)
	out := make([]intersectedList, n)
	for i := 0; i < n; i++ {
		// offsets[i] is the offset for i-th posting list. It gets decremented as we
		// iterate over buckets.
		out[i].offset = int(ts.Offset)
		out[i].ulist = &task.List{Uids: []uint64{}}
	}

	// Iterate over every bucket in TokensTable.
	t := posting.GetTokensTable(attr)

	var token string
	if ts.Desc {
		token = t.GetLast()
	} else {
		token = t.GetFirst()
	}
BUCKETS:
	for len(token) > 0 {
		err := intersectBucket(ts, attr, token, out)
		switch err {
		case errDone:
			break BUCKETS
		case errContinue:
			// Continue iterating over tokens.
		default:
			return &emptySortResult, err
		}
		if ts.Desc {
			token = t.GetPrev(token)
		} else {
			token = t.GetNext(token)
		}
	}

	r := new(task.SortResult)
	for _, il := range out {
		r.UidMatrix = append(r.UidMatrix, il.ulist)
	}
	return r, nil
}
Beispiel #16
0
func (n *node) AddToCluster(ctx context.Context, pid uint64) error {
	addr := n.peers.Get(pid)
	x.AssertTruef(len(addr) > 0, "Unable to find conn pool for peer: %d", pid)
	rc := &task.RaftContext{
		Addr:  addr,
		Group: n.raftContext.Group,
		Id:    pid,
	}
	rcBytes, err := rc.Marshal()
	x.Check(err)
	return n.raft.ProposeConfChange(ctx, raftpb.ConfChange{
		ID:      pid,
		Type:    raftpb.ConfChangeAddNode,
		NodeID:  pid,
		Context: rcBytes,
	})
}
Beispiel #17
0
// returns true if the geometry represented by g is within the given loop or cap
func (q QueryData) isWithin(g types.Geo) bool {
	x.AssertTruef(q.pt != nil || q.loop != nil || q.cap != nil, "At least a point, loop or cap should be defined.")
	gpt, ok := g.T.(*geom.Point)
	if !ok {
		// We will only consider points for within queries.
		return false
	}

	s2pt := pointFromPoint(gpt)
	if q.pt != nil {
		return q.pt.ApproxEqual(s2pt)
	}

	if q.loop != nil {
		return q.loop.ContainsPoint(s2pt)
	}
	return q.cap.ContainsPoint(s2pt)
}
Beispiel #18
0
// AssignUidsOverNetwork assigns new uids and writes them to the umap.
func AssignUidsOverNetwork(ctx context.Context, umap map[string]uint64) error {
	gid := group.BelongsTo("_uid_")
	num := createNumQuery(gid, umap)

	var ul *task.List
	var err error
	if groups().ServesGroup(gid) {
		ul, err = assignUids(ctx, num)
		if err != nil {
			return err
		}

	} else {
		_, addr := groups().Leader(gid)
		p := pools().get(addr)
		conn, err := p.Get()
		if err != nil {
			x.TraceError(ctx, x.Wrapf(err, "Error while retrieving connection"))
			return err
		}
		defer p.Put(conn)

		c := NewWorkerClient(conn)
		ul, err = c.AssignUids(ctx, num)
		if err != nil {
			x.TraceError(ctx, x.Wrapf(err, "Error while getting uids"))
			return err
		}
	}

	x.AssertTruef(len(ul.Uids) == int(num.Val),
		"Requested: %d != Retrieved Uids: %d", num.Val, len(ul.Uids))

	i := 0
	for k, v := range umap {
		if v == 0 {
			uid := ul.Uids[i]
			umap[k] = uid // Write uids to map.
			i++
		}
	}
	return nil
}
Beispiel #19
0
func parsePredicates(groupId uint32, p string) error {
	preds := strings.Split(p, ",")
	x.AssertTruef(len(preds) > 0, "Length of predicates in config should be > 0")

	for _, pred := range preds {
		pred = strings.TrimSpace(pred)
		meta := predMeta{
			val: pred,
			gid: groupId,
		}
		if strings.HasSuffix(pred, "*") {
			meta.val = strings.TrimSuffix(meta.val, "*")
		} else {
			meta.exactMatch = true
		}
		groupConfig.pred = append(groupConfig.pred, meta)
	}
	return nil
}
Beispiel #20
0
// returns true if the geometry represented by uid/attr contains the given point
func (q QueryData) contains(g types.Geo) bool {
	x.AssertTruef(q.pt != nil || q.loop != nil, "At least a point or loop should be defined.")
	if q.loop != nil {
		// We don't support polygons containing polygons yet.
		return false
	}

	poly, ok := g.T.(*geom.Polygon)
	if !ok {
		// We will only consider polygons for contains queries.
		return false
	}

	s2loop, err := loopFromPolygon(poly)
	if err != nil {
		return false
	}
	return s2loop.ContainsPoint(*q.pt)
}
Beispiel #21
0
// AddMutationWithIndex is AddMutation with support for indexing.
func (l *List) AddMutationWithIndex(ctx context.Context, t *task.DirectedEdge, op uint32) error {
	x.AssertTruef(len(t.Attr) > 0,
		"[%s] [%d] [%v] %d %d\n", t.Attr, t.Entity, t.Value, t.ValueId, op)

	var vbytes []byte
	var vtype byte
	var verr error

	doUpdateIndex := pstore != nil && (t.Value != nil) &&
		schema.IsIndexed(t.Attr)
	if doUpdateIndex {
		// Check last posting for original value BEFORE any mutation actually happens.
		vbytes, vtype, verr = l.Value()
	}
	hasMutated, err := l.AddMutation(ctx, t, op)
	if err != nil {
		return err
	}
	if !hasMutated || !doUpdateIndex {
		return nil
	}

	// Exact matches.
	if verr == nil && len(vbytes) > 0 {
		delTerm := vbytes
		delType := vtype
		p := types.ValueForType(types.TypeID(delType))
		if err := p.UnmarshalBinary(delTerm); err != nil {
			return err
		}
		addIndexMutations(ctx, t.Attr, t.Entity, p, true)
	}
	if op == Set {
		p := types.ValueForType(types.TypeID(t.ValueType))
		if err := p.UnmarshalBinary(t.Value); err != nil {
			return err
		}
		addIndexMutations(ctx, t.Attr, t.Entity, p, false)
	}
	return nil
}
Beispiel #22
0
// addIndexMutations adds mutation(s) for a single term, to maintain index.
func addIndexMutations(ctx context.Context, attr string, uid uint64,
	p types.Value, del bool) {
	x.AssertTrue(uid != 0)
	tokens, err := IndexTokens(attr, p)
	if err != nil {
		// This data is not indexable
		return
	}
	edge := &task.DirectedEdge{
		ValueId: uid,
		Attr:    attr,
		Label:   "idx",
	}

	tokensTable := GetTokensTable(attr)
	x.AssertTruef(tokensTable != nil, "TokensTable missing for attr %s", attr)

	for _, token := range tokens {
		addIndexMutation(ctx, attr, token, tokensTable, edge, del)
	}
}
Beispiel #23
0
// newGraph returns the SubGraph and its task query.
func newGraph(ctx context.Context, gq *gql.GraphQuery) (*SubGraph, error) {
	euid, exid := gq.UID, gq.XID
	// This would set the Result field in SubGraph,
	// and populate the children for attributes.
	if len(exid) > 0 {
		x.AssertTruef(!strings.HasPrefix(exid, "_new_:"), "Query shouldn't contain _new_")
		euid = farm.Fingerprint64([]byte(exid))
		x.Trace(ctx, "Xid: %v Uid: %v", exid, euid)
	}

	if euid == 0 && gq.Func == nil {
		err := x.Errorf("Invalid query, query internal id is zero and generator is nil")
		x.TraceError(ctx, err)
		return nil, err
	}

	// For the root, the name to be used in result is stored in Alias, not Attr.
	// The attr at root (if present) would stand for the source functions attr.
	args := params{
		AttrType: schema.TypeOf(gq.Alias),
		isDebug:  gq.Alias == "debug",
		Alias:    gq.Alias,
	}

	sg := &SubGraph{
		Params: args,
	}
	if gq.Func != nil {
		sg.Attr = gq.Func.Attr
		sg.SrcFunc = append(sg.SrcFunc, gq.Func.Name)
		sg.SrcFunc = append(sg.SrcFunc, gq.Func.Args...)
	}
	if euid > 0 {
		// euid is the root UID.
		sg.SrcUIDs = &task.List{Uids: []uint64{euid}}
		sg.uidMatrix = []*task.List{&task.List{Uids: []uint64{euid}}}
	}
	sg.values = createNilValuesList(1)
	return sg, nil
}
Beispiel #24
0
func (n *node) joinPeers() {
	// Get leader information for MY group.
	pid, paddr := groups().Leader(n.gid)
	n.Connect(pid, paddr)
	fmt.Printf("Connected with: %v\n", paddr)

	addr := n.peers.Get(pid)
	pool := pools().get(addr)
	x.AssertTruef(pool != nil, "Unable to find addr for peer: %d", pid)

	// Bring the instance up to speed first.
	_, err := populateShard(n.ctx, pool, 0)
	x.Checkf(err, "Error while populating shard")

	conn, err := pool.Get()
	x.Check(err)
	defer pool.Put(conn)

	c := NewWorkerClient(conn)
	x.Printf("Calling JoinCluster")
	_, err = c.JoinCluster(n.ctx, n.raftContext)
	x.Checkf(err, "Error while joining cluster")
	x.Printf("Done with JoinCluster call\n")
}
Beispiel #25
0
// GetTokens returns the corresponding index keys based on the type
// of function.
func GetTokens(funcArgs []string) ([]string, *QueryData, error) {
	x.AssertTruef(len(funcArgs) > 1, "Invalid function")
	funcName := strings.ToLower(funcArgs[0])
	switch funcName {
	case "near":
		if len(funcArgs) != 3 {
			return nil, nil, x.Errorf("near function requires 3 arguments, but got %d",
				len(funcArgs))
		}
		maxDist, err := strconv.ParseFloat(funcArgs[2], 64)
		if err != nil {
			return nil, nil, x.Wrapf(err, "Error while converting distance to float")
		}
		return queryTokens(QueryTypeNear, funcArgs[1], maxDist)
	case "within":
		if len(funcArgs) != 2 {
			return nil, nil, x.Errorf("within function requires 2 arguments, but got %d",
				len(funcArgs))
		}
		return queryTokens(QueryTypeWithin, funcArgs[1], 0.0)
	case "contains":
		if len(funcArgs) != 2 {
			return nil, nil, x.Errorf("contains function requires 2 arguments, but got %d",
				len(funcArgs))
		}
		return queryTokens(QueryTypeContains, funcArgs[1], 0.0)
	case "intersects":
		if len(funcArgs) != 2 {
			return nil, nil, x.Errorf("intersects function requires 2 arguments, but got %d",
				len(funcArgs))
		}
		return queryTokens(QueryTypeIntersects, funcArgs[1], 0.0)
	default:
		return nil, nil, x.Errorf("Invalid geo function")
	}
}
Beispiel #26
0
func main() {
	x.Init()

	fin, err := os.Open(filename)
	x.Check(err)
	defer fin.Close()

	scanner := bufio.NewScanner(fin)
	var numLines, numValues, numNames, numReleaseDates int
	graph = make(map[string]map[uint64][]uint64)
	gNames = make(map[uint64]string)
	gReleaseDates = make(map[uint64]string)

	for scanner.Scan() {
		numLines++
		tokens := strings.Split(scanner.Text(), "\t")
		x.AssertTruef(len(tokens) == 4, scanner.Text())

		src := tokens[0]
		x.AssertTrue(bracketed(src))
		src = removeFirstLast(src)
		srcUID := farm.Fingerprint64([]byte(src))

		pred := tokens[1]
		x.AssertTrue(bracketed(pred))
		pred = removeFirstLast(pred)

		value := tokens[2]

		if bracketed(value) {
			// Normal edge.
			value = removeFirstLast(value)
			destUID := farm.Fingerprint64([]byte(value))
			m, found := graph[pred]
			if !found {
				m = make(map[uint64][]uint64)
				graph[pred] = m
			}
			m[srcUID] = append(m[srcUID], destUID)
		} else {
			numValues++

			// Check for "@".
			pos := strings.LastIndex(value, "@")
			if pos >= 0 {
				pred = pred + "." + value[pos+1:]
				value = removeFirstLast(value[:pos])
			}

			if pred == "type.object.name.en" {
				numNames++
				gNames[srcUID] = value
			} else if pred == "film.film.initial_release_date" {
				numReleaseDates++
				gReleaseDates[srcUID] = value
			}
		}
	}

	fmt.Printf("Num lines read: %d\n", numLines)
	fmt.Printf("Num predicates: %d\n", len(graph))
	fmt.Printf("Num values read: %d\n", numValues)
	fmt.Printf("Num names read: %d\n", numNames)
	fmt.Printf("Num release dates read: %d\n", numReleaseDates)

	x.AssertTrue(numLines > 0)
	x.AssertTrue(len(graph) > 0)
	x.AssertTrue(numValues > 0)
	x.AssertTrue(numNames > 0)
	x.AssertTrue(numReleaseDates > 0)

	//	doFilterString()
	//	doSortRelease()
	doGen()
}
Beispiel #27
0
func (s *filterTreeStack) peek() *FilterTree {
	x.AssertTruef(!s.empty(), "Trying to peek empty stack")
	return s.a[len(s.a)-1]
}
Beispiel #28
0
// parseFilter parses the filter directive to produce a QueryFilter / parse tree.
func parseFilter(l *lex.Lexer) (*FilterTree, error) {
	item := <-l.Items
	if item.Typ != itemLeftRound {
		return nil, x.Errorf("Expected ( after filter directive")
	}

	opStack := new(filterTreeStack)
	opStack.push(&FilterTree{Op: "("}) // Push ( onto operator stack.
	valueStack := new(filterTreeStack)

	for item = range l.Items {
		if item.Typ == itemFilterFunc { // Value.
			f := &Function{}
			leaf := &FilterTree{Func: f}
			f.Name = item.Val
			itemInFunc := <-l.Items
			if itemInFunc.Typ != itemLeftRound {
				return nil, x.Errorf("Expected ( after func name [%s]", leaf.Func.Name)
			}
			var terminated bool
			for itemInFunc = range l.Items {
				if itemInFunc.Typ == itemRightRound {
					terminated = true
					break
				} else if itemInFunc.Typ != itemFilterFuncArg {
					return nil, x.Errorf("Expected arg after func [%s], but got item %v",
						leaf.Func.Name, itemInFunc)
				}
				if len(f.Attr) == 0 {
					f.Attr = itemInFunc.Val
				} else {
					f.Args = append(f.Args, itemInFunc.Val)
				}
			}
			if !terminated {
				return nil, x.Errorf("Expected ) to terminate func definition")
			}
			valueStack.push(leaf)

		} else if item.Typ == itemLeftRound { // Just push to op stack.
			opStack.push(&FilterTree{Op: "("})

		} else if item.Typ == itemRightRound { // Pop op stack until we see a (.
			for !opStack.empty() {
				topOp := opStack.peek()
				if topOp.Op == "(" {
					break
				}
				evalStack(opStack, valueStack)
			}
			opStack.pop() // Pop away the (.
			if opStack.empty() {
				// The parentheses are balanced out. Let's break.
				break
			}

		} else if item.Typ == itemFilterAnd || item.Typ == itemFilterOr {
			op := "&"
			if item.Typ == itemFilterOr {
				op = "|"
			}
			opPred := filterOpPrecedence[op]
			x.AssertTruef(opPred > 0, "Expected opPred > 0: %d", opPred)
			// Evaluate the stack until we see an operator with strictly lower pred.
			for !opStack.empty() {
				topOp := opStack.peek()
				if filterOpPrecedence[topOp.Op] < opPred {
					break
				}
				evalStack(opStack, valueStack)
			}
			opStack.push(&FilterTree{Op: op}) // Push current operator.
		}
	}

	// For filters, we start with ( and end with ). We expect to break out of loop
	// when the parentheses balance off, and at that point, opStack should be empty.
	// For other applications, typically after all items are
	// consumed, we will run a loop like "while opStack is nonempty, evalStack".
	// This is not needed here.
	x.AssertTruef(opStack.empty(), "Op stack should be empty when we exit")

	if valueStack.empty() {
		// This happens when we have @filter(). We can either return an error or
		// ignore. Currently, let's just ignore and pretend there is no filter.
		return nil, nil
	}

	if valueStack.size() != 1 {
		return nil, x.Errorf("Expected one item in value stack, but got %d",
			valueStack.size())
	}
	return valueStack.pop(), nil
}
Beispiel #29
0
func main() {
	x.Init()

	fin, err := os.Open(filename)
	x.Check(err)
	defer fin.Close()

	scanner := bufio.NewScanner(fin)
	var numLines, numValues, numNames, numReleaseDates int
	invGraph = make(map[string]map[uint64][]uint64)

	for scanner.Scan() {
		numLines++
		tokens := strings.Split(scanner.Text(), "\t")
		x.AssertTruef(len(tokens) == 4, scanner.Text())

		src := tokens[0]
		x.AssertTrue(bracketed(src))
		src = removeFirstLast(src)
		srcUID := farm.Fingerprint64([]byte(src))

		pred := tokens[1]
		x.AssertTrue(bracketed(pred))
		pred = removeFirstLast(pred)

		value := tokens[2]

		if bracketed(value) {
			// Normal edge.
			value = removeFirstLast(value)
			destUID := farm.Fingerprint64([]byte(value))
			m, found := invGraph[pred]
			if !found {
				m = make(map[uint64][]uint64)
				invGraph[pred] = m
			}
			// We are building an inverse map!
			m[destUID] = append(m[destUID], srcUID)
		} else {
			// A value.
			numValues++
			value = removeFirstLast(value)
			if pred == "type.object.name" {
				numNames++

				// Do some custom processing here.
				value = strings.ToLower(value)
				vTokens := strings.Split(value, " ")
				var found bool
				for _, t := range vTokens {
					if t == "the" {
						found = true
						break
					}
				}
				if found {
					goodUIDs = append(goodUIDs, srcUID)
				}

			} else if pred == "film.film.initial_release_date" {
				numReleaseDates++
			}
		}
	}

	fmt.Printf("Num lines read: %d\n", numLines)
	fmt.Printf("Num predicates: %d\n", len(invGraph))
	fmt.Printf("Num values read: %d\n", numValues)
	fmt.Printf("Num names read: %d\n", numNames)
	fmt.Printf("Num release dates read: %d\n", numReleaseDates)
	fmt.Printf("Num good UIDs: %d\n", len(goodUIDs))

	x.AssertTrue(numLines > 0)
	x.AssertTrue(len(invGraph) > 0)
	x.AssertTrue(numValues > 0)
	x.AssertTrue(numNames > 0)
	x.AssertTrue(numReleaseDates > 0)
	x.AssertTrue(len(goodUIDs) > 0)

	doGood()
}
Beispiel #30
0
// queryTokens returns the tokens to be used to look up the geo index for a given filter.
func queryTokens(qt QueryType, data string, maxDistance float64) ([]string, *QueryData, error) {
	// Try to parse the data as geo type.
	var g types.Geo
	geoData := strings.Replace(data, "'", "\"", -1)
	err := g.UnmarshalText([]byte(geoData))
	if err != nil {
		return nil, nil, x.Wrapf(err, "Cannot decode given geoJson input")
	}

	var l *s2.Loop
	var pt *s2.Point
	switch v := g.T.(type) {
	case *geom.Point:
		p := pointFromPoint(v)
		pt = &p

	case *geom.Polygon:
		l, err = loopFromPolygon(v)
		if err != nil {
			return nil, nil, err
		}

	default:
		return nil, nil, x.Errorf("Cannot query using a geometry of type %T", v)
	}

	x.AssertTruef(l != nil || pt != nil, "We should have a point or a loop.")

	parents, cover, err := indexCells(g)
	if err != nil {
		return nil, nil, err
	}

	switch qt {
	case QueryTypeWithin:
		// For a within query we only need to look at the objects whose parents match our cover.
		// So we take our cover and prefix with the parentPrefix to look in the index.
		toks := toTokens(cover, parentPrefix)
		return toks, &QueryData{pt: pt, loop: l, qtype: qt}, nil

	case QueryTypeContains:
		if l != nil {
			return nil, nil, x.Errorf("Cannot use a polygon in a contains query")
		}
		// For a contains query, we only need to look at the objects whose cover matches our
		// parents. So we take our parents and prefix with the coverPrefix to look in the index.
		return toTokens(parents, coverPrefix), &QueryData{pt: pt, qtype: qt}, nil

	case QueryTypeNear:
		if l != nil {
			return nil, nil, x.Errorf("Cannot use a polygon in a near query")
		}
		return nearQueryKeys(*pt, maxDistance)

	case QueryTypeIntersects:
		// An intersects query is essentially the union of contains and within. So we look at all
		// the objects whose parents match our cover as well as all the objects whose cover matches
		// our parents.
		toks := parentCoverTokens(parents, cover)
		return toks, &QueryData{pt: pt, loop: l, qtype: qt}, nil

	default:
		return nil, nil, x.Errorf("Unknown query type")
	}
}