Beispiel #1
0
func ExampleServer() {
	store.Get().Init("/tmp/ldb_" + x.UniqueString(10))
	search.Get().Init("memsearch")
	indexer.Register("EntityKind", SimpleIndexer{})

	server := indexer.NewServer(100, 5)
	server.InfiniteLoop(30 * time.Minute)
	// This would never exit.
	// OR, you could also just run this once, if you're
	// testing your setup.
	server.LoopOnce()
	server.Finish() // Finish is only useful when you're looping once.
}
Beispiel #2
0
func AddDocs(e search.Engine) {
	for idx, name := range galaxies {
		var d x.Doc
		d.Id = x.UniqueString(5)
		d.Kind = "Galaxy"
		d.NanoTs = time.Now().UnixNano()
		m := make(map[string]interface{})
		m["name"] = name
		m["pos"] = idx
		d.Data = m

		if err := e.Update(d); err != nil {
			log.Fatalf("While updating: %v", err)
			return
		}
	}
}
Beispiel #3
0
func (l *Leveldb) Commit(its []*x.Instruction) error {
	var keys []string
	for _, it := range its {
		var key string
		for m := 0; m < 10; m++ {
			key = fmt.Sprintf("%s_%s", it.SubjectId, x.UniqueString(5))
			log.WithField("key", key).Debug("Checking existence of key")
			if has, err := l.db.Has([]byte(key), nil); err != nil {
				x.LogErr(log, err).WithField("key", key).Error("While check if key exists")
				continue
			} else if has {
				continue
			} else {
				break
			}
			log.Errorf("Exhausted %d tries", m)
			return errors.New("Exhausted tries")
		}
		log.WithField("key", key).Debug("Is unique")
		keys = append(keys, key)
	}

	b := new(leveldb.Batch)
	for idx, it := range its {
		key := []byte(keys[idx])
		buf, err := it.GobEncode()
		if err != nil {
			x.LogErr(log, err).Error("While encoding")
			return err
		}
		b.Put(key, buf)
	}
	if err := l.db.Write(b, nil); err != nil {
		x.LogErr(log, err).Error("While writing to db")
		return err
	}
	log.Debugf("%d instructions committed", len(its))

	return nil
}
Beispiel #4
0
func main() {
	rand.Seed(0) // Keep output consistent.
	flag.Parse()
	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
	} else {
		logrus.SetLevel(logrus.ErrorLevel)
	}

	c = req.NewContextWithUpdates(10, 1000) // 62^10 permutations

	// Initialize leveldb.
	dirname, err := ioutil.TempDir("", "ldb_")
	if err != nil {
		log.Fatalf("While creating temp directory: %v\n", err)
		return
	}
	defer os.RemoveAll(dirname)
	store.Get().Init(dirname)

	// Initialize Elasticsearch.
	// search.Get().Init("http://192.168.59.103:9200")

	// Other possible initializations. Remember to import the right driver.
	// store.Get().Init("mysql", "root@tcp(127.0.0.1:3306)/test", "instructions")
	// store.Get().Init("cassone", "crudtest", "instructions")
	// store.Get().Init("192.168.59.103:27017", "crudtest", "instructions")
	// store.Get().Init("192.168.59.103:28015", "test", "instructions")

	search.Get().Init("memsearch")
	indexer.Register("Post", SimpleIndexer{})
	indexer.Register("Like", SimpleIndexer{})
	indexer.Register("Comment", SimpleIndexer{})
	indexer.Run(c, 2)
	defer indexer.WaitForDone(c)

	log.Debug("Store initialized. Checking search...")
	uid := newUser()

	// Let's get started. User 'uid' creates a new Post.
	// This Post shares a url, adds some text and some tags.
	tags := [3]string{"search", "cat", "videos"}
	err = store.NewUpdate("User", uid).SetSource(uid).AddChild("Post").
		Set("url", "www.google.com").Set("body", "You can search for cat videos here").
		Set("tags", tags).Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	// Now let's add a comment and two likes to our new post.
	// One user would add a comment and one like. Another user would
	// just like the post.
	//
	// It's best to have the same 'source' for one set of operations.
	// In REST APIs, this is how things would always be. Each REST call
	// is from one user (and never two different users).
	// This way the creation of like "entity", and the properties
	// of that new like entity have the same source.
	//
	// So, here's Step 1: A new user would add a comment, and like the post.
	fmt.Print("Added a new post by user")
	user := printAndGetUser(uid)
	post := user.Post[0]

	p := store.NewUpdate("Post", post.Id).SetSource(newUser())
	p.AddChild("Like").Set("thumb", 1)
	p.AddChild("Comment").Set("body",
		fmt.Sprintf("Comment %s on the post", x.UniqueString(2)))
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	// Step 2: Another user would now like the post.
	p = store.NewUpdate("Post", post.Id).SetSource(newUser())
	p.AddChild("Like").Set("thumb", 1)
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Added a Comment and 2 Likes on Post")

	user = printAndGetUser(uid)
	post = user.Post[0]
	if len(post.Comment) == 0 {
		log.Fatalf("No comment found: %+v", post)
	}
	comment := post.Comment[0]

	// Now another user likes and replies to the comment that was added above.
	// So, it's a comment within a comment.
	p = store.NewUpdate("Comment", comment.Id).SetSource(newUser())
	p.AddChild("Like").Set("thumb", 1)
	p.AddChild("Comment").Set("body",
		fmt.Sprintf("Comment %s on comment", x.UniqueString(2)))
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Added a Comment and a Like on Comment")
	user = printAndGetUser(uid)
	post = user.Post[0]
	if len(post.Comment) == 0 {
		log.Fatalf("No comment found: %+v", post)
	}
	comment = post.Comment[0]
	if len(comment.Like) == 0 {
		log.Fatalf("No like found: %+v", comment)
	}
	like := comment.Like[0]

	// So far we have this structure:
	// User
	//  L Post
	//         L 2 * Like
	//         L Comment
	//            L Comment
	//            L Like

	// This is what most social platforms do. But, let's go
	// one level further, and also comment on the Likes on Comment.
	// User
	//    L Post
	//         L 2 * Like
	//         L Comment
	//            L Comment
	//            L Like
	//                 L Comment

	// Another user Comments on the Like on Comment on Post.

	p = store.NewUpdate("Like", like.Id).SetSource(newUser()).
		AddChild("Comment").Set("body",
		fmt.Sprintf("Comment %s on Like", x.UniqueString(2)))
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Added Comment on Like")
	user = printAndGetUser(uid)

	{
		docs, err := search.Get().NewQuery("Like").Order("data.source").Run()
		if err != nil {
			x.LogErr(log, err).Fatal("While searching for Post")
			return
		}
		for _, doc := range docs {
			log.WithField("doc", doc).Debug("Resulting doc")
		}
		log.Debug("Search query over")
	}

	post = user.Post[0]
	if len(post.Comment) == 0 {
		log.Fatalf("No comment found: %+v", post)
	}
	comment = post.Comment[0]
	p = store.NewUpdate("Comment", comment.Id).SetSource(newUser()).Set("censored", true)
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	q := store.NewQuery(comment.Id).UptoDepth(0)
	result, err := q.Run()
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Set censored=true on comment")
	prettyPrintResult(*result)

	user = printAndGetUser(uid)
	post = user.Post[0]
	if pid, err := store.Parent(post.Id); err == nil {
		if pid != user.Id {
			log.Fatal("Post's parent id doesn't match user id.")
			return
		}
		log.WithFields(logrus.Fields{
			"id":        post.Id,
			"parent_id": pid,
			"user_id":   user.Id,
		}).Debug("Parent id matches")
	} else {
		log.Fatal(err.Error())
		return
	}

	if len(post.Like) == 0 {
		log.Fatalf("No like found: %+v", post)
	}
	like = post.Like[0]
	p = store.NewUpdate("Like", like.Id).SetSource(newUser()).MarkDeleted()
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	q = store.NewQuery(uid).Collect("Post")
	q.Collect("Like").UptoDepth(10)
	q.Collect("Comment").UptoDepth(10).FilterOut("censored")
	result, err = q.Run()
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Filter out censored Comment and mark one Like as deleted.")
	prettyPrintResult(*result)
	// By now we have a fairly complex Post structure. CRUD for
	// which would have been a lot of work to put together using
	// typical SQL / NoSQL tables.

	{
		ch := make(chan x.Entity, 10)
		done := make(chan bool)
		go processChannel(ch, done)
		num, last, err := store.Get().Iterate("", 100, ch)
		if err != nil {
			x.LogErr(log, err).Fatal("While iterating")
			return
		}
		fmt.Printf("Found %d results\n", num)
		fmt.Printf("Last Entity: %+v\n", last)
		close(ch)
		<-done
	}

	{
		fmt.Println()
		fmt.Println()
		fmt.Print("Searching for doc with url = www.google.com")
		q := search.Get().NewQuery("Post").Order("-data.activity")
		q.NewAndFilter().AddExact("data.url", "www.google.com")
		docs, err := q.Run()
		if err != nil {
			x.LogErr(log, err).Fatal("While searching for Post")
			return
		}
		for _, doc := range docs {
			js, err := json.MarshalIndent(doc, "", "'  ")
			if err != nil {
				log.Fatalf("While marshal: %v\n", err)
				return
			}
			fmt.Printf("\n%s\n%s\n%s\n\n", sep1, string(js), sep2)
			// log.WithField("doc", doc).Debug("Resulting doc")
		}
		log.Debug("Search query over")
	}
}
Beispiel #5
0
func newUser() string {
	return "uid_" + x.UniqueString(3)
}
Beispiel #6
0
func ExampleUniqueString() {
	u := x.UniqueString(3)
	fmt.Println(len(u))
	// Output: 3
}
Beispiel #7
0
func (n *Update) doExecute(c *req.Context, its *[]*x.Instruction) error {
	for pred, val := range n.edges {
		if len(n.source) == 0 {
			return errors.New(fmt.Sprintf(
				"No source specified for id: %v kind: %v", n.id, n.kind))
		}

		i := new(x.Instruction)
		i.SubjectId = n.id
		i.SubjectType = n.kind
		i.Predicate = pred

		if b, err := json.Marshal(val); err != nil {
			return err
		} else {
			i.Object = b
		}
		i.Source = n.source
		i.NanoTs = n.NanoTs
		log.WithField("instruction", i).Debug("Pushing to list")
		*its = append(*its, i)
	}

	if len(n.children) == 0 {
		return nil
	}
	if len(n.source) == 0 {
		return errors.New(fmt.Sprintf(
			"No source specified for id: %v kind: %v", n.id, n.kind))
	}

	// Children can only be added, not deleted via API. But they can be stopped
	// from being retrieved.
	// Scenario: How do I stop childA from being retrieved?
	// Answer:
	// Modify child by adding a 'deleted' edge
	// Get(ChildKind, ChildId).Set("deleted", true).Execute(c)
	//
	// Then for retrieval from parent:
	// NewQuery(ParentKind, ParentId).Collect(ChildKind).FilterOut("deleted")
	// This would remove all children with a 'deleted' edge.
	// Better still
	// Get(ChildKind, ChildId).MarkDeleted().Execute(c)
	// would automatically filter out that child and it's children
	// from being retrieved.

	for _, child := range n.children {
		if len(child.id) > 0 {
			log.WithField("child_id", child.id).Fatal(
				"Child id should be empty for all current use cases")
			return errors.New("Non empty child id")
		}

		for idx := 0; ; idx++ { // Retry loop.
			child.id = x.UniqueString(c.NumCharsUnique)
			log.WithField("id", child.id).Debug("Checking availability of new id")
			if isnew := Get().IsNew(child.id); isnew {
				log.WithField("id", child.id).Debug("New id available")
				break
			}
			if idx >= 30 {
				return errors.New("Unable to find new id")
			}
		}
		// Create edge from parent to child
		i := new(x.Instruction)
		i.SubjectId = n.id
		i.SubjectType = n.kind
		i.Predicate = child.kind
		i.ObjectId = child.id
		i.Source = n.source
		i.NanoTs = n.NanoTs
		log.WithField("instruction", i).Debug("Pushing to list")
		*its = append(*its, i)

		// Create edge from child to parent
		i = new(x.Instruction)
		i.SubjectId = child.id
		i.SubjectType = child.kind
		i.Predicate = "_parent_"
		i.ObjectId = n.id
		i.Source = n.source
		i.NanoTs = n.NanoTs
		log.WithField("instruction", i).Debug("Pushing to list")
		*its = append(*its, i)

		if err := child.doExecute(c, its); err != nil {
			return err
		}
	}
	return nil
}