func ExampleServer() {
store.Get().Init("/tmp/ldb_" + x.UniqueString(10))
search.Get().Init("memsearch")
indexer.Register("EntityKind", SimpleIndexer{})
server := indexer.NewServer(100, 5)
server.InfiniteLoop(30 * time.Minute)
// This would never exit.
// OR, you could also just run this once, if you're
// testing your setup.
server.LoopOnce()
server.Finish() // Finish is only useful when you're looping once.
}
func AddDocs(e search.Engine) {
for idx, name := range galaxies {
var d x.Doc
d.Id = x.UniqueString(5)
d.Kind = "Galaxy"
d.NanoTs = time.Now().UnixNano()
m := make(map[string]interface{})
m["name"] = name
m["pos"] = idx
d.Data = m
if err := e.Update(d); err != nil {
log.Fatalf("While updating: %v", err)
return
}
}
}
func (l *Leveldb) Commit(its []*x.Instruction) error {
var keys []string
for _, it := range its {
var key string
for m := 0; m < 10; m++ {
key = fmt.Sprintf("%s_%s", it.SubjectId, x.UniqueString(5))
log.WithField("key", key).Debug("Checking existence of key")
if has, err := l.db.Has([]byte(key), nil); err != nil {
x.LogErr(log, err).WithField("key", key).Error("While check if key exists")
continue
} else if has {
continue
} else {
break
}
log.Errorf("Exhausted %d tries", m)
return errors.New("Exhausted tries")
}
log.WithField("key", key).Debug("Is unique")
keys = append(keys, key)
}
b := new(leveldb.Batch)
for idx, it := range its {
key := []byte(keys[idx])
buf, err := it.GobEncode()
if err != nil {
x.LogErr(log, err).Error("While encoding")
return err
}
b.Put(key, buf)
}
if err := l.db.Write(b, nil); err != nil {
x.LogErr(log, err).Error("While writing to db")
return err
}
log.Debugf("%d instructions committed", len(its))
return nil
}
func main() {
rand.Seed(0) // Keep output consistent.
flag.Parse()
if *debug {
logrus.SetLevel(logrus.DebugLevel)
} else {
logrus.SetLevel(logrus.ErrorLevel)
}
c = req.NewContextWithUpdates(10, 1000) // 62^10 permutations
// Initialize leveldb.
dirname, err := ioutil.TempDir("", "ldb_")
if err != nil {
log.Fatalf("While creating temp directory: %v\n", err)
return
}
defer os.RemoveAll(dirname)
store.Get().Init(dirname)
// Initialize Elasticsearch.
// search.Get().Init("http://192.168.59.103:9200")
// Other possible initializations. Remember to import the right driver.
// store.Get().Init("mysql", "root@tcp(127.0.0.1:3306)/test", "instructions")
// store.Get().Init("cassone", "crudtest", "instructions")
// store.Get().Init("192.168.59.103:27017", "crudtest", "instructions")
// store.Get().Init("192.168.59.103:28015", "test", "instructions")
search.Get().Init("memsearch")
indexer.Register("Post", SimpleIndexer{})
indexer.Register("Like", SimpleIndexer{})
indexer.Register("Comment", SimpleIndexer{})
indexer.Run(c, 2)
defer indexer.WaitForDone(c)
log.Debug("Store initialized. Checking search...")
uid := newUser()
// Let's get started. User 'uid' creates a new Post.
// This Post shares a url, adds some text and some tags.
tags := [3]string{"search", "cat", "videos"}
err = store.NewUpdate("User", uid).SetSource(uid).AddChild("Post").
Set("url", "www.google.com").Set("body", "You can search for cat videos here").
Set("tags", tags).Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
// Now let's add a comment and two likes to our new post.
// One user would add a comment and one like. Another user would
// just like the post.
//
// It's best to have the same 'source' for one set of operations.
// In REST APIs, this is how things would always be. Each REST call
// is from one user (and never two different users).
// This way the creation of like "entity", and the properties
// of that new like entity have the same source.
//
// So, here's Step 1: A new user would add a comment, and like the post.
fmt.Print("Added a new post by user")
user := printAndGetUser(uid)
post := user.Post[0]
p := store.NewUpdate("Post", post.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
p.AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on the post", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
// Step 2: Another user would now like the post.
p = store.NewUpdate("Post", post.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added a Comment and 2 Likes on Post")
user = printAndGetUser(uid)
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment := post.Comment[0]
// Now another user likes and replies to the comment that was added above.
// So, it's a comment within a comment.
p = store.NewUpdate("Comment", comment.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
p.AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on comment", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added a Comment and a Like on Comment")
user = printAndGetUser(uid)
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment = post.Comment[0]
if len(comment.Like) == 0 {
log.Fatalf("No like found: %+v", comment)
}
like := comment.Like[0]
// So far we have this structure:
// User
// L Post
// L 2 * Like
// L Comment
// L Comment
// L Like
// This is what most social platforms do. But, let's go
// one level further, and also comment on the Likes on Comment.
// User
// L Post
// L 2 * Like
// L Comment
// L Comment
// L Like
// L Comment
// Another user Comments on the Like on Comment on Post.
p = store.NewUpdate("Like", like.Id).SetSource(newUser()).
AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on Like", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added Comment on Like")
user = printAndGetUser(uid)
{
docs, err := search.Get().NewQuery("Like").Order("data.source").Run()
if err != nil {
x.LogErr(log, err).Fatal("While searching for Post")
return
}
for _, doc := range docs {
log.WithField("doc", doc).Debug("Resulting doc")
}
log.Debug("Search query over")
}
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment = post.Comment[0]
p = store.NewUpdate("Comment", comment.Id).SetSource(newUser()).Set("censored", true)
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
q := store.NewQuery(comment.Id).UptoDepth(0)
result, err := q.Run()
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Set censored=true on comment")
prettyPrintResult(*result)
user = printAndGetUser(uid)
post = user.Post[0]
if pid, err := store.Parent(post.Id); err == nil {
if pid != user.Id {
log.Fatal("Post's parent id doesn't match user id.")
return
}
log.WithFields(logrus.Fields{
"id": post.Id,
"parent_id": pid,
"user_id": user.Id,
}).Debug("Parent id matches")
} else {
log.Fatal(err.Error())
return
}
if len(post.Like) == 0 {
log.Fatalf("No like found: %+v", post)
}
like = post.Like[0]
p = store.NewUpdate("Like", like.Id).SetSource(newUser()).MarkDeleted()
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
q = store.NewQuery(uid).Collect("Post")
q.Collect("Like").UptoDepth(10)
q.Collect("Comment").UptoDepth(10).FilterOut("censored")
result, err = q.Run()
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Filter out censored Comment and mark one Like as deleted.")
prettyPrintResult(*result)
// By now we have a fairly complex Post structure. CRUD for
// which would have been a lot of work to put together using
// typical SQL / NoSQL tables.
{
ch := make(chan x.Entity, 10)
done := make(chan bool)
go processChannel(ch, done)
num, last, err := store.Get().Iterate("", 100, ch)
if err != nil {
x.LogErr(log, err).Fatal("While iterating")
return
}
fmt.Printf("Found %d results\n", num)
fmt.Printf("Last Entity: %+v\n", last)
close(ch)
<-done
}
{
fmt.Println()
fmt.Println()
fmt.Print("Searching for doc with url = www.google.com")
q := search.Get().NewQuery("Post").Order("-data.activity")
q.NewAndFilter().AddExact("data.url", "www.google.com")
docs, err := q.Run()
if err != nil {
x.LogErr(log, err).Fatal("While searching for Post")
return
}
for _, doc := range docs {
js, err := json.MarshalIndent(doc, "", "' ")
if err != nil {
log.Fatalf("While marshal: %v\n", err)
return
}
fmt.Printf("\n%s\n%s\n%s\n\n", sep1, string(js), sep2)
// log.WithField("doc", doc).Debug("Resulting doc")
}
log.Debug("Search query over")
}
}
func newUser() string {
return "uid_" + x.UniqueString(3)
}
func ExampleUniqueString() {
u := x.UniqueString(3)
fmt.Println(len(u))
// Output: 3
}
func (n *Update) doExecute(c *req.Context, its *[]*x.Instruction) error {
for pred, val := range n.edges {
if len(n.source) == 0 {
return errors.New(fmt.Sprintf(
"No source specified for id: %v kind: %v", n.id, n.kind))
}
i := new(x.Instruction)
i.SubjectId = n.id
i.SubjectType = n.kind
i.Predicate = pred
if b, err := json.Marshal(val); err != nil {
return err
} else {
i.Object = b
}
i.Source = n.source
i.NanoTs = n.NanoTs
log.WithField("instruction", i).Debug("Pushing to list")
*its = append(*its, i)
}
if len(n.children) == 0 {
return nil
}
if len(n.source) == 0 {
return errors.New(fmt.Sprintf(
"No source specified for id: %v kind: %v", n.id, n.kind))
}
// Children can only be added, not deleted via API. But they can be stopped
// from being retrieved.
// Scenario: How do I stop childA from being retrieved?
// Answer:
// Modify child by adding a 'deleted' edge
// Get(ChildKind, ChildId).Set("deleted", true).Execute(c)
//
// Then for retrieval from parent:
// NewQuery(ParentKind, ParentId).Collect(ChildKind).FilterOut("deleted")
// This would remove all children with a 'deleted' edge.
// Better still
// Get(ChildKind, ChildId).MarkDeleted().Execute(c)
// would automatically filter out that child and it's children
// from being retrieved.
for _, child := range n.children {
if len(child.id) > 0 {
log.WithField("child_id", child.id).Fatal(
"Child id should be empty for all current use cases")
return errors.New("Non empty child id")
}
for idx := 0; ; idx++ { // Retry loop.
child.id = x.UniqueString(c.NumCharsUnique)
log.WithField("id", child.id).Debug("Checking availability of new id")
if isnew := Get().IsNew(child.id); isnew {
log.WithField("id", child.id).Debug("New id available")
break
}
if idx >= 30 {
return errors.New("Unable to find new id")
}
}
// Create edge from parent to child
i := new(x.Instruction)
i.SubjectId = n.id
i.SubjectType = n.kind
i.Predicate = child.kind
i.ObjectId = child.id
i.Source = n.source
i.NanoTs = n.NanoTs
log.WithField("instruction", i).Debug("Pushing to list")
*its = append(*its, i)
// Create edge from child to parent
i = new(x.Instruction)
i.SubjectId = child.id
i.SubjectType = child.kind
i.Predicate = "_parent_"
i.ObjectId = n.id
i.Source = n.source
i.NanoTs = n.NanoTs
log.WithField("instruction", i).Debug("Pushing to list")
*its = append(*its, i)
if err := child.doExecute(c, its); err != nil {
return err
}
}
return nil
}