func ExampleServer() {
store.Get().Init("/tmp/ldb_" + x.UniqueString(10))
search.Get().Init("memsearch")
indexer.Register("EntityKind", SimpleIndexer{})
server := indexer.NewServer(100, 5)
server.InfiniteLoop(30 * time.Minute)
// This would never exit.
// OR, you could also just run this once, if you're
// testing your setup.
server.LoopOnce()
server.Finish() // Finish is only useful when you're looping once.
}
func AddDocs(e search.Engine) {
for idx, name := range galaxies {
var d x.Doc
d.Id = x.UniqueString(5)
d.Kind = "Galaxy"
d.NanoTs = time.Now().UnixNano()
m := make(map[string]interface{})
m["name"] = name
m["pos"] = idx
d.Data = m
if err := e.Update(d); err != nil {
log.Fatalf("While updating: %v", err)
return
}
}
}
func (l *Leveldb) Commit(_ string, its []*x.Instruction) error {
var keys []string
for _, it := range its {
var key string
for m := 0; m < 10; m++ {
key = fmt.Sprintf("%s_%s", it.SubjectId, x.UniqueString(5))
log.WithField("key", key).Debug("Checking existence of key")
if has, err := l.db.Has([]byte(key), nil); err != nil {
x.LogErr(log, err).WithField("key", key).Error("While check if key exists")
continue
} else if has {
continue
} else {
break
}
log.Errorf("Exhausted %d tries", m)
return errors.New("Exhausted tries")
}
log.WithField("key", key).Debug("Is unique")
keys = append(keys, key)
}
b := new(leveldb.Batch)
for idx, it := range its {
key := []byte(keys[idx])
buf, err := it.GobEncode()
if err != nil {
x.LogErr(log, err).Error("While encoding")
return err
}
b.Put(key, buf)
}
if err := l.db.Write(b, nil); err != nil {
x.LogErr(log, err).Error("While writing to db")
return err
}
log.Debugf("%d instructions committed", len(its))
return nil
}
func main() {
rand.Seed(time.Now().UnixNano())
fmt.Println("Running...")
ctx = new(req.Context)
l := new(store.Leveldb)
l.SetBloomFilter(13)
ctx.Store = l
ctx.Store.Init("leveldb", "test")
e := echo.New()
e.SetDebug(true)
e.Use(mw.Recover())
e.Static("/assets/", "public/assets")
e.Get("/webms/:name", func(c *echo.Context) error {
resp := c.Response()
name := c.Param("name")
resp.Header().Add("Content-Type", "video/webm")
http.ServeFile(resp, c.Request(), "webms/"+name)
return nil
})
e.Static("/webms/", "webms")
//e.ServeFile("/", "public/index.html")
e.Get("/", func(c *echo.Context) error {
http.ServeFile(c.Response(), c.Request(), "public/index.html")
return nil
})
e.ServeFile("/upload", "public/upload.html")
e.Post("/upload", func(c *echo.Context) error {
req := c.Request()
name := req.FormValue("name")
ttags := req.FormValue("tags")
tags := strings.Split(ttags, " ")
path := x.UniqueString(10)
// Read files
file := req.MultipartForm.File["file"]
src, err := file[0].Open()
if err != nil {
return err
}
defer src.Close()
// Destination file
dst, err := os.Create("webms/" + path + ".webm")
if err != nil {
return err
}
defer dst.Close()
if _, err = io.Copy(dst, src); err != nil {
return err
}
if err = api.Get("WebmCont", rootid).SetSource(rootid).AddChild("Webm").Set("path", path).
Set("tags", tags).Set("name", name).Execute(ctx); err != nil {
return err
}
return c.String(http.StatusOK, "eyo uploaded file")
})
e.Get("/webm", func(c *echo.Context) error {
result, err := api.NewQuery("WebmCont", rootid).Collect("Webm").Run(ctx)
if err != nil {
return err
}
by, err := result.ToJson()
if err != nil {
return err
}
return c.String(200, string(by))
})
e.Get("/webm/filter/:filter", func(c *echo.Context) error {
tags := strings.Split(c.Param("filter"), "+")
result, err := api.NewQuery("WebmCont", rootid).Collect("Webm").Run(ctx)
if err != nil {
return err
}
by, err := filter(result, tags...)
if err != nil {
return err
}
return c.String(200, string(by))
})
e.Get("/webm/:id", func(c *echo.Context) error {
result, err := api.NewQuery("Webm", c.Param("id")).Run(ctx)
if err != nil {
return err
}
by, err := result.ToJson()
if err != nil {
return err
}
return c.String(200, string(by))
})
e.Get("/webm/:id/tag/:tag", func(c *echo.Context) error {
result, err := api.NewQuery("Webm", c.Param("id")).Run(ctx)
if err != nil {
return err
}
var tags []interface{}
tagb, ok := result.Columns["tags"].Value.([]interface{})
if ok {
tags = tagb
}
cont := contains(tags, c.Param("tag"))
var atags []string
var resp string
if cont {
resp = "removed"
atags = convertButLeave(tags, c.Param("tag"))
} else {
resp = "added"
atags = convert(tags)
atags = append(atags, c.Param("tag"))
}
err = api.Get("Webm", c.Param("id")).SetSource(rootid).Set("tags", atags).Execute(ctx)
if err != nil {
return err
}
return c.String(200, resp)
})
graceful.ListenAndServe(e.Server(":8080"), 5*time.Second)
}
func main() {
rand.Seed(time.Now().UnixNano())
fmt.Println("Running...")
flag.Parse()
c = new(req.Context)
if *storeType == "leveldb" {
l := new(store.Leveldb)
l.SetBloomFilter(13)
c.Store = l
c.Store.Init(*storeType, "/tmp/ldb_"+x.UniqueString(10))
} else if *storeType == "cass" {
cluster := gocql.NewCluster("192.168.59.103")
cluster.Keyspace = "crudtest"
cluster.Consistency = gocql.Quorum
cass := new(store.Cassandra)
if session, err := cluster.CreateSession(); err != nil {
panic(err)
} else {
cass.SetSession(session)
}
c.Store = cass
c.Store.Init(*storeType, "instructions")
} else if *storeType == "mysql" {
db, err := sql.Open("mysql", "root@tcp(127.0.0.1:3306)/test")
if err != nil {
panic(err)
}
if err = db.Ping(); err != nil {
panic(err)
}
log.Info("Connection to mysql successful")
sqldb := new(store.Sql)
sqldb.SetDb(db)
c.Store = sqldb
c.Store.Init(*storeType, "instructions")
} else if *storeType == "postgres" {
db, err := sql.Open("postgres", "postgres://localhost/test?sslmode=disable")
if err != nil {
panic(err)
}
if err = db.Ping(); err != nil {
panic(err)
}
log.Info("Connection to postgres successful")
sqldb := new(store.Sql)
sqldb.SetDb(db)
c.Store = sqldb
c.Store.Init(*storeType, "instructions")
} else if *storeType == "rethinkdb" {
session, err := r.Connect(r.ConnectOpts{
// Address: "192.168.59.103:28015",
Address: "localhost:28015",
Database: "crudtest",
})
if err != nil {
panic(err)
}
rethinkdb := new(store.RethinkDB)
rethinkdb.SetSession(session)
c.Store = rethinkdb
c.Store.Init(*storeType, "instructions")
} else if *storeType == "mongodb" {
// "192.168.59.103:27017"
session, err := mgo.Dial("localhost:27017")
if err != nil {
panic(err)
}
session.SetMode(mgo.Monotonic, true)
mongodb := new(store.MongoDB)
mongodb.SetSession(session, "crudtest")
c.Store = mongodb
c.Store.Init(*storeType, "instructions")
} else if *storeType == "datastore" {
c.TablePrefix = "Test-"
c.Store = new(store.Datastore)
c.Store.Init(*storeType, "gce-project-id")
} else {
panic("Invalid store")
}
var err error
uid := newUser()
// Let's get started. User 'uid' creates a new Post.
// This Post shares a url, adds some text and some tags.
tags := [3]string{"search", "cat", "videos"}
err = api.Get("User", uid).SetSource(uid).AddChild("Post").
Set("url", "www.google.com").Set("body", "You can search for cat videos here").
Set("tags", tags).Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Stored Post")
// Now let's add a comment and two likes to our new post.
// One user would add a comment and one like. Another user would
// just like the post.
//
// It's best to have the same 'source' for one set of operations.
// In REST APIs, this is how things would always be. Each REST call
// is from one user (and never two different users).
// This way the creation of like "entity", and the properties
// of that new like entity have the same source.
//
// So, here's Step 1: A new user would add a comment, and like the post.
user := printAndGetUser(uid)
post := user.Post[0]
p := api.Get("Post", post.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
p.AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on the post", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
// Step 2: Another user would now like the post.
p = api.Get("Post", post.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added 1 Comment and 2 Like on Post")
user = printAndGetUser(uid)
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment := post.Comment[0]
// Now another user likes and replies to the comment that was added above.
// So, it's a comment within a comment.
p = api.Get("Comment", comment.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
p.AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on comment", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added Comment on Comment")
user = printAndGetUser(uid)
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment = post.Comment[0]
if len(comment.Like) == 0 {
log.Fatalf("No like found: %+v", comment)
}
like := comment.Like[0]
// So far we have this structure:
// User
// L Post
// L 2 * Like
// L Comment
// L Comment
// L Like
// This is what most social platforms do. But, let's go
// one level further, and also comment on the Likes on Comment.
// User
// L Post
// L 2 * Like
// L Comment
// L Comment
// L Like
// L Comment
// Another user Comments on the Like on Comment on Post.
p = api.Get("Like", like.Id).SetSource(newUser()).
AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on Like", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added Comment on Like")
user = printAndGetUser(uid)
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment = post.Comment[0]
p = api.Get("Comment", comment.Id).SetSource(newUser()).Set("censored", true)
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
q := api.NewQuery("Comment", comment.Id).UptoDepth(0)
result, err := q.Run(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
js, err := result.ToJson()
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Printf("\n%s\n%s\n%s\n", sep, string(js), sep)
user = printAndGetUser(uid)
post = user.Post[0]
if len(post.Like) == 0 {
log.Fatalf("No like found: %+v", post)
}
like = post.Like[0]
p = api.Get("Like", like.Id).SetSource(newUser()).MarkDeleted()
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
q = api.NewQuery("User", uid).Collect("Post")
q.Collect("Like").UptoDepth(10)
q.Collect("Comment").UptoDepth(10).FilterOut("censored")
result, err = q.Run(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
js, err = result.ToJson()
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Printf("\n%s\n%s\n%s\n", sep, string(js), sep)
// By now we have a fairly complex Post structure. CRUD for
// which would have been a lot of work to put together using
// typical SQL / NoSQL tables.
}
func newUser() string {
return "uid_" + x.UniqueString(3)
}
func (n *Update) doExecute(c *req.Context, its *[]*x.Instruction) error {
for pred, val := range n.edges {
if len(n.source) == 0 {
return errors.New(fmt.Sprintf(
"No source specified for id: %v kind: %v", n.id, n.kind))
}
i := new(x.Instruction)
i.SubjectId = n.id
i.SubjectType = n.kind
i.Predicate = pred
if b, err := json.Marshal(val); err != nil {
return err
} else {
i.Object = b
}
i.Source = n.source
i.NanoTs = n.NanoTs
log.WithField("instruction", i).Debug("Pushing to list")
*its = append(*its, i)
}
if len(n.children) == 0 {
return nil
}
if len(n.source) == 0 {
return errors.New(fmt.Sprintf(
"No source specified for id: %v kind: %v", n.id, n.kind))
}
// Children can only be added, not deleted via API. But they can be stopped
// from being retrieved.
// Scenario: How do I stop childA from being retrieved?
// Answer:
// Modify child by adding a 'deleted' edge
// Get(ChildKind, ChildId).Set("deleted", true).Execute(c)
//
// Then for retrieval from parent:
// NewQuery(ParentKind, ParentId).Collect(ChildKind).FilterOut("deleted")
// This would remove all children with a 'deleted' edge.
// Better still
// Get(ChildKind, ChildId).MarkDeleted().Execute(c)
// would automatically filter out that child and it's children
// from being retrieved.
for _, child := range n.children {
if len(child.id) > 0 {
log.WithField("child_id", child.id).Fatal(
"Child id should be empty for all current use cases")
return errors.New("Non empty child id")
}
for idx := 0; ; idx++ { // Retry loop.
child.id = x.UniqueString(c.NumCharsUnique)
log.WithField("id", child.id).Debug("Checking availability of new id")
if isnew := Get().IsNew(child.id); isnew {
log.WithField("id", child.id).Debug("New id available")
break
}
if idx >= 30 {
return errors.New("Unable to find new id")
}
}
// Create edge from parent to child
i := new(x.Instruction)
i.SubjectId = n.id
i.SubjectType = n.kind
i.Predicate = child.kind
i.ObjectId = child.id
i.Source = n.source
i.NanoTs = n.NanoTs
log.WithField("instruction", i).Debug("Pushing to list")
*its = append(*its, i)
// Create edge from child to parent
i = new(x.Instruction)
i.SubjectId = child.id
i.SubjectType = child.kind
i.Predicate = "_parent_"
i.ObjectId = n.id
i.Source = n.source
i.NanoTs = n.NanoTs
log.WithField("instruction", i).Debug("Pushing to list")
*its = append(*its, i)
if err := child.doExecute(c, its); err != nil {
return err
}
}
return nil
}
func ExampleUniqueString() {
u := x.UniqueString(3)
fmt.Println(len(u))
// Output: 3
}
func main() {
rand.Seed(0) // Keep output consistent.
flag.Parse()
if *debug {
logrus.SetLevel(logrus.DebugLevel)
} else {
logrus.SetLevel(logrus.ErrorLevel)
}
c = req.NewContextWithUpdates(10, 1000) // 62^10 permutations
// Initialize leveldb.
dirname, err := ioutil.TempDir("", "ldb_")
if err != nil {
log.Fatalf("While creating temp directory: %v\n", err)
return
}
defer os.RemoveAll(dirname)
store.Get().Init(dirname)
// Initialize Elasticsearch.
// search.Get().Init("http://192.168.59.103:9200")
// Other possible initializations. Remember to import the right driver.
// store.Get().Init("mysql", "root@tcp(127.0.0.1:3306)/test", "instructions")
// store.Get().Init("cassone", "crudtest", "instructions")
// store.Get().Init("192.168.59.103:27017", "crudtest", "instructions")
// store.Get().Init("192.168.59.103:28015", "test", "instructions")
search.Get().Init("memsearch")
indexer.Register("Post", SimpleIndexer{})
indexer.Register("Like", SimpleIndexer{})
indexer.Register("Comment", SimpleIndexer{})
indexer.Run(c, 2)
defer indexer.WaitForDone(c)
log.Debug("Store initialized. Checking search...")
uid := newUser()
// Let's get started. User 'uid' creates a new Post.
// This Post shares a url, adds some text and some tags.
tags := [3]string{"search", "cat", "videos"}
err = store.NewUpdate("User", uid).SetSource(uid).AddChild("Post").
Set("url", "www.google.com").Set("body", "You can search for cat videos here").
Set("tags", tags).Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
// Now let's add a comment and two likes to our new post.
// One user would add a comment and one like. Another user would
// just like the post.
//
// It's best to have the same 'source' for one set of operations.
// In REST APIs, this is how things would always be. Each REST call
// is from one user (and never two different users).
// This way the creation of like "entity", and the properties
// of that new like entity have the same source.
//
// So, here's Step 1: A new user would add a comment, and like the post.
fmt.Print("Added a new post by user")
user := printAndGetUser(uid)
post := user.Post[0]
p := store.NewUpdate("Post", post.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
p.AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on the post", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
// Step 2: Another user would now like the post.
p = store.NewUpdate("Post", post.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added a Comment and 2 Likes on Post")
user = printAndGetUser(uid)
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment := post.Comment[0]
// Now another user likes and replies to the comment that was added above.
// So, it's a comment within a comment.
p = store.NewUpdate("Comment", comment.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
p.AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on comment", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added a Comment and a Like on Comment")
user = printAndGetUser(uid)
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment = post.Comment[0]
if len(comment.Like) == 0 {
log.Fatalf("No like found: %+v", comment)
}
like := comment.Like[0]
// So far we have this structure:
// User
// L Post
// L 2 * Like
// L Comment
// L Comment
// L Like
// This is what most social platforms do. But, let's go
// one level further, and also comment on the Likes on Comment.
// User
// L Post
// L 2 * Like
// L Comment
// L Comment
// L Like
// L Comment
// Another user Comments on the Like on Comment on Post.
p = store.NewUpdate("Like", like.Id).SetSource(newUser()).
AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on Like", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added Comment on Like")
user = printAndGetUser(uid)
{
docs, err := search.Get().NewQuery("Like").Order("data.source").Run()
if err != nil {
x.LogErr(log, err).Fatal("While searching for Post")
return
}
for _, doc := range docs {
log.WithField("doc", doc).Debug("Resulting doc")
}
log.Debug("Search query over")
}
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment = post.Comment[0]
p = store.NewUpdate("Comment", comment.Id).SetSource(newUser()).Set("censored", true)
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
q := store.NewQuery(comment.Id).UptoDepth(0)
result, err := q.Run()
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Set censored=true on comment")
prettyPrintResult(*result)
user = printAndGetUser(uid)
post = user.Post[0]
if pid, err := store.Parent(post.Id); err == nil {
if pid != user.Id {
log.Fatal("Post's parent id doesn't match user id.")
return
}
log.WithFields(logrus.Fields{
"id": post.Id,
"parent_id": pid,
"user_id": user.Id,
}).Debug("Parent id matches")
} else {
log.Fatal(err.Error())
return
}
if len(post.Like) == 0 {
log.Fatalf("No like found: %+v", post)
}
like = post.Like[0]
p = store.NewUpdate("Like", like.Id).SetSource(newUser()).MarkDeleted()
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
q = store.NewQuery(uid).Collect("Post")
q.Collect("Like").UptoDepth(10)
q.Collect("Comment").UptoDepth(10).FilterOut("censored")
result, err = q.Run()
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Filter out censored Comment and mark one Like as deleted.")
prettyPrintResult(*result)
// By now we have a fairly complex Post structure. CRUD for
// which would have been a lot of work to put together using
// typical SQL / NoSQL tables.
{
ch := make(chan x.Entity, 10)
done := make(chan bool)
go processChannel(ch, done)
num, last, err := store.Get().Iterate("", 100, ch)
if err != nil {
x.LogErr(log, err).Fatal("While iterating")
return
}
fmt.Printf("Found %d results\n", num)
fmt.Printf("Last Entity: %+v\n", last)
close(ch)
<-done
}
{
fmt.Println()
fmt.Println()
fmt.Print("Searching for doc with url = www.google.com")
q := search.Get().NewQuery("Post").Order("-data.activity")
q.NewAndFilter().AddExact("data.url", "www.google.com")
docs, err := q.Run()
if err != nil {
x.LogErr(log, err).Fatal("While searching for Post")
return
}
for _, doc := range docs {
js, err := json.MarshalIndent(doc, "", "' ")
if err != nil {
log.Fatalf("While marshal: %v\n", err)
return
}
fmt.Printf("\n%s\n%s\n%s\n\n", sep1, string(js), sep2)
// log.WithField("doc", doc).Debug("Resulting doc")
}
log.Debug("Search query over")
}
}