func processUpdates(c *req.Context) {
defer wg.Done()
for entity := range c.Updates {
idxr, pok := Get(entity.Kind)
if !pok {
continue
}
dirty := idxr.OnUpdate(entity)
for _, de := range dirty {
didxr, dok := Get(de.Kind)
if !dok {
continue
}
doc := didxr.Regenerate(de)
log.WithField("doc", doc).Debug("Regenerated doc")
if search.Get() == nil {
continue
}
err := search.Get().Update(doc)
if err != nil {
x.LogErr(log, err).WithField("doc", doc).
Error("While updating in search engine")
}
}
}
log.Info("Finished processing channel")
}
func ExampleSearch() {
path, err := ioutil.TempDir("", "gocrudldb_")
if err != nil {
x.LogErr(log, err).Fatal("Opening file")
return
}
store.Get().Init(path) // leveldb
search.Get().Init() // memsearch
// Run indexer to update entities in search engine in real time.
c := req.NewContextWithUpdates(10, 100)
indexer.Register("Child", SimpleIndexer{})
indexer.Run(c, 2)
u := store.NewUpdate("Root", "bigbang").SetSource("author")
for i := 0; i < 10; i++ {
child := u.AddChild("Child").Set("pos", i).Set("particle", particles[i])
if i == 5 {
child.MarkDeleted() // This shouldn't be retrieved anymore.
}
}
if err = u.Execute(c); err != nil {
x.LogErr(log, err).Fatal("While updating")
return
}
indexer.WaitForDone(c) // Block until indexing is done.
docs, err := search.Get().NewQuery("Child").Order("-data.pos").Run()
if err != nil {
x.LogErr(log, err).Fatal("While searching")
return
}
fmt.Println("docs:", len(docs))
for _, doc := range docs {
m := doc.Data.(map[string]interface{})
fmt.Println(m["pos"], m["particle"])
}
// Output:
// docs: 9
// 9 higgs boson
// 8 boson
// 7 photon
// 6 bottom
// 4 down
// 3 gluon
// 2 top
// 1 charm
// 0 up
}
func main() {
rand.Seed(time.Now().UnixNano())
flag.Parse()
if *eip == "" {
flag.Usage()
return
}
engine := search.Get()
engine.Init("http://" + *eip + ":9200")
r := rand.Intn(100)
uid := fmt.Sprintf("uid_%d", r)
var au Author
au.Id = fmt.Sprintf("mrjn-%d", r)
au.Ts = r
doc := x.Doc{Kind: "test", Id: uid, NanoTs: time.Now().UnixNano(), Data: au}
if err := engine.Update(doc); err != nil {
fmt.Printf("Error: %v\n", err)
return
}
q := engine.NewQuery("test").Order("-Data.Ts").Limit(*num)
q.NewAndFilter().AddExact("Data.Id", "mrjn")
docs, err := q.Run()
if err != nil {
fmt.Printf("Error: %v\n", err)
return
}
for _, doc := range docs {
fmt.Printf("Doc: %+v\n", doc)
}
return
}
// NewServer returns back a server which runs continously in
// a loop to find and re-index entities stored.
// You can control the amount of memory consumed by the server
// via buffer of pending entities in the channel, and the
// rate of processing of these entities via numRoutines.
func NewServer(buffer int, numRoutines int) *Server {
if search.Get() == nil {
log.Fatal("No search engine found")
}
s := new(Server)
s.ch = make(chan x.Entity, buffer)
s.wg = new(sync.WaitGroup)
for i := 0; i < numRoutines; i++ {
s.wg.Add(1)
go s.regenerateAndIndex()
}
return s
}
func ExampleServer() {
store.Get().Init("/tmp/ldb_" + x.UniqueString(10))
search.Get().Init("memsearch")
indexer.Register("EntityKind", SimpleIndexer{})
server := indexer.NewServer(100, 5)
server.InfiniteLoop(30 * time.Minute)
// This would never exit.
// OR, you could also just run this once, if you're
// testing your setup.
server.LoopOnce()
server.Finish() // Finish is only useful when you're looping once.
}
func (s *Server) regenerateAndIndex() {
defer s.wg.Done()
for entity := range s.ch {
idxr, ok := Get(entity.Kind)
if !ok {
continue
}
doc := idxr.Regenerate(entity)
log.WithField("doc", doc).Debug("Regenerated doc")
if err := search.Get().Update(doc); err != nil {
x.LogErr(log, err).WithField("doc", doc).
Error("While updating in search engine")
}
}
}
func main() {
rand.Seed(0) // Keep output consistent.
flag.Parse()
if *debug {
logrus.SetLevel(logrus.DebugLevel)
} else {
logrus.SetLevel(logrus.ErrorLevel)
}
c = req.NewContextWithUpdates(10, 1000) // 62^10 permutations
// Initialize leveldb.
dirname, err := ioutil.TempDir("", "ldb_")
if err != nil {
log.Fatalf("While creating temp directory: %v\n", err)
return
}
defer os.RemoveAll(dirname)
store.Get().Init(dirname)
// Initialize Elasticsearch.
// search.Get().Init("http://192.168.59.103:9200")
// Other possible initializations. Remember to import the right driver.
// store.Get().Init("mysql", "root@tcp(127.0.0.1:3306)/test", "instructions")
// store.Get().Init("cassone", "crudtest", "instructions")
// store.Get().Init("192.168.59.103:27017", "crudtest", "instructions")
// store.Get().Init("192.168.59.103:28015", "test", "instructions")
search.Get().Init("memsearch")
indexer.Register("Post", SimpleIndexer{})
indexer.Register("Like", SimpleIndexer{})
indexer.Register("Comment", SimpleIndexer{})
indexer.Run(c, 2)
defer indexer.WaitForDone(c)
log.Debug("Store initialized. Checking search...")
uid := newUser()
// Let's get started. User 'uid' creates a new Post.
// This Post shares a url, adds some text and some tags.
tags := [3]string{"search", "cat", "videos"}
err = store.NewUpdate("User", uid).SetSource(uid).AddChild("Post").
Set("url", "www.google.com").Set("body", "You can search for cat videos here").
Set("tags", tags).Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
// Now let's add a comment and two likes to our new post.
// One user would add a comment and one like. Another user would
// just like the post.
//
// It's best to have the same 'source' for one set of operations.
// In REST APIs, this is how things would always be. Each REST call
// is from one user (and never two different users).
// This way the creation of like "entity", and the properties
// of that new like entity have the same source.
//
// So, here's Step 1: A new user would add a comment, and like the post.
fmt.Print("Added a new post by user")
user := printAndGetUser(uid)
post := user.Post[0]
p := store.NewUpdate("Post", post.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
p.AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on the post", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
// Step 2: Another user would now like the post.
p = store.NewUpdate("Post", post.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added a Comment and 2 Likes on Post")
user = printAndGetUser(uid)
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment := post.Comment[0]
// Now another user likes and replies to the comment that was added above.
// So, it's a comment within a comment.
p = store.NewUpdate("Comment", comment.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
p.AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on comment", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added a Comment and a Like on Comment")
user = printAndGetUser(uid)
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment = post.Comment[0]
if len(comment.Like) == 0 {
log.Fatalf("No like found: %+v", comment)
}
like := comment.Like[0]
// So far we have this structure:
// User
// L Post
// L 2 * Like
// L Comment
// L Comment
// L Like
// This is what most social platforms do. But, let's go
// one level further, and also comment on the Likes on Comment.
// User
// L Post
// L 2 * Like
// L Comment
// L Comment
// L Like
// L Comment
// Another user Comments on the Like on Comment on Post.
p = store.NewUpdate("Like", like.Id).SetSource(newUser()).
AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on Like", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added Comment on Like")
user = printAndGetUser(uid)
{
docs, err := search.Get().NewQuery("Like").Order("data.source").Run()
if err != nil {
x.LogErr(log, err).Fatal("While searching for Post")
return
}
for _, doc := range docs {
log.WithField("doc", doc).Debug("Resulting doc")
}
log.Debug("Search query over")
}
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment = post.Comment[0]
p = store.NewUpdate("Comment", comment.Id).SetSource(newUser()).Set("censored", true)
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
q := store.NewQuery(comment.Id).UptoDepth(0)
result, err := q.Run()
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Set censored=true on comment")
prettyPrintResult(*result)
user = printAndGetUser(uid)
post = user.Post[0]
if pid, err := store.Parent(post.Id); err == nil {
if pid != user.Id {
log.Fatal("Post's parent id doesn't match user id.")
return
}
log.WithFields(logrus.Fields{
"id": post.Id,
"parent_id": pid,
"user_id": user.Id,
}).Debug("Parent id matches")
} else {
log.Fatal(err.Error())
return
}
if len(post.Like) == 0 {
log.Fatalf("No like found: %+v", post)
}
like = post.Like[0]
p = store.NewUpdate("Like", like.Id).SetSource(newUser()).MarkDeleted()
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
q = store.NewQuery(uid).Collect("Post")
q.Collect("Like").UptoDepth(10)
q.Collect("Comment").UptoDepth(10).FilterOut("censored")
result, err = q.Run()
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Filter out censored Comment and mark one Like as deleted.")
prettyPrintResult(*result)
// By now we have a fairly complex Post structure. CRUD for
// which would have been a lot of work to put together using
// typical SQL / NoSQL tables.
{
ch := make(chan x.Entity, 10)
done := make(chan bool)
go processChannel(ch, done)
num, last, err := store.Get().Iterate("", 100, ch)
if err != nil {
x.LogErr(log, err).Fatal("While iterating")
return
}
fmt.Printf("Found %d results\n", num)
fmt.Printf("Last Entity: %+v\n", last)
close(ch)
<-done
}
{
fmt.Println()
fmt.Println()
fmt.Print("Searching for doc with url = www.google.com")
q := search.Get().NewQuery("Post").Order("-data.activity")
q.NewAndFilter().AddExact("data.url", "www.google.com")
docs, err := q.Run()
if err != nil {
x.LogErr(log, err).Fatal("While searching for Post")
return
}
for _, doc := range docs {
js, err := json.MarshalIndent(doc, "", "' ")
if err != nil {
log.Fatalf("While marshal: %v\n", err)
return
}
fmt.Printf("\n%s\n%s\n%s\n\n", sep1, string(js), sep2)
// log.WithField("doc", doc).Debug("Resulting doc")
}
log.Debug("Search query over")
}
}