func TestVersions(t *testing.T) {
path, err := ioutil.TempDir("", "gocrudldb_")
if err != nil {
t.Fatal("Opening leveldb file")
return
}
store.Get().Init(path) // leveldb
c := req.NewContext(10)
var d int
for d = 660; d < 670; d++ {
if err = store.NewUpdate("Ticker", "GOOG").SetSource("nasdaq").
Set("price", d).Execute(c); err != nil {
t.Errorf("When updating store: %+v", err)
t.Fail()
}
}
result, err := store.NewQuery("GOOG").Run()
if err != nil {
t.Errorf("When querying store: %+v", err)
t.Fail()
}
versions, present := result.Columns["price"]
if !present {
t.Errorf("Column price should be present: %+v", result)
t.Fail()
}
if versions.Count() != 10 {
t.Errorf("Num count expected: 10. Got: %v", versions.Count())
}
if versions.Latest().Value.(float64) != 669 {
t.Errorf("Latest value expected 669. Got: %v", versions.Latest())
}
if versions.Oldest().Value.(float64) != 660 {
t.Errorf("Oldest value expected 660. Got: %v", versions.Oldest())
}
type jval struct {
Kind string `json:"kind,omitempty"`
Id string `json:"id,omitempty"`
Source string `json:"modifier,omitempty"`
Price int `json:"price,omitempty"`
}
js, err := result.ToJson()
if err != nil {
t.Errorf("While converting to JSON: %v", err)
}
t.Log(string(js))
var jv jval
if err = json.Unmarshal(js, &jv); err != nil {
t.Errorf("While unmarshal to struct: %v", err)
}
if jv.Price != 669 {
t.Errorf("Latest value expected 669. Got: %+v", jv)
}
if jv.Source != "nasdaq" {
t.Errorf("Source expected nasdaq. Got: %+v", jv)
}
}
func ExampleServer() {
store.Get().Init("/tmp/ldb_" + x.UniqueString(10))
search.Get().Init("memsearch")
indexer.Register("EntityKind", SimpleIndexer{})
server := indexer.NewServer(100, 5)
server.InfiniteLoop(30 * time.Minute)
// This would never exit.
// OR, you could also just run this once, if you're
// testing your setup.
server.LoopOnce()
server.Finish() // Finish is only useful when you're looping once.
}
func ExampleSearch() {
path, err := ioutil.TempDir("", "gocrudldb_")
if err != nil {
x.LogErr(log, err).Fatal("Opening file")
return
}
store.Get().Init(path) // leveldb
search.Get().Init() // memsearch
// Run indexer to update entities in search engine in real time.
c := req.NewContextWithUpdates(10, 100)
indexer.Register("Child", SimpleIndexer{})
indexer.Run(c, 2)
u := store.NewUpdate("Root", "bigbang").SetSource("author")
for i := 0; i < 10; i++ {
child := u.AddChild("Child").Set("pos", i).Set("particle", particles[i])
if i == 5 {
child.MarkDeleted() // This shouldn't be retrieved anymore.
}
}
if err = u.Execute(c); err != nil {
x.LogErr(log, err).Fatal("While updating")
return
}
indexer.WaitForDone(c) // Block until indexing is done.
docs, err := search.Get().NewQuery("Child").Order("-data.pos").Run()
if err != nil {
x.LogErr(log, err).Fatal("While searching")
return
}
fmt.Println("docs:", len(docs))
for _, doc := range docs {
m := doc.Data.(map[string]interface{})
fmt.Println(m["pos"], m["particle"])
}
// Output:
// docs: 9
// 9 higgs boson
// 8 boson
// 7 photon
// 6 bottom
// 4 down
// 3 gluon
// 2 top
// 1 charm
// 0 up
}
func ExampleStore() {
path, err := ioutil.TempDir("", "gocrudldb_")
if err != nil {
x.LogErr(log, err).Fatal("Opening file")
return
}
store.Get().Init(path) // leveldb
// Update some data.
c := req.NewContext(10) // 62^10 permutations
err = store.NewUpdate("Root", "bigbang").SetSource("author").
Set("when", "13.8 billion years ago").Set("explosive", true).Execute(c)
if err != nil {
x.LogErr(log, err).Fatal("Commiting update")
return
}
// Retrieve that data
result, err := store.NewQuery("bigbang").Run()
if err != nil {
x.LogErr(log, err).Fatal("While querying store")
return
}
fmt.Println(result.Kind) // Root
fmt.Println(result.Id) // bigbang
data := result.ToMap()
{
val, ok := data["explosive"]
if !ok {
log.Fatal("creator should be set")
return
}
fmt.Println(val) // true
}
{
val, ok := data["when"]
if !ok {
log.Fatal("creator should be set")
return
}
fmt.Println(val)
}
// Output:
// Root
// bigbang
// true
// 13.8 billion years ago
}
// LoopOnce would cycle over all entities in the store, and re-index them.
func (s *Server) LoopOnce() {
var total uint64
from := ""
for {
found, last, err := store.Get().Iterate(from, 1000, s.ch)
if err != nil {
x.LogErr(log, err).Error("While iterating")
return
}
if found == 0 {
log.WithField("total", total).Info("Reached end of cycle")
return
}
log.WithFields(logrus.Fields{
"num_processed": found,
"last": last,
}).Debug("Iteration chunk done")
total += uint64(found)
from = last.Id
}
log.Fatal("This should never be reached.")
return
}
func main() {
rand.Seed(0) // Keep output consistent.
flag.Parse()
if *debug {
logrus.SetLevel(logrus.DebugLevel)
} else {
logrus.SetLevel(logrus.ErrorLevel)
}
c = req.NewContextWithUpdates(10, 1000) // 62^10 permutations
// Initialize leveldb.
dirname, err := ioutil.TempDir("", "ldb_")
if err != nil {
log.Fatalf("While creating temp directory: %v\n", err)
return
}
defer os.RemoveAll(dirname)
store.Get().Init(dirname)
// Initialize Elasticsearch.
// search.Get().Init("http://192.168.59.103:9200")
// Other possible initializations. Remember to import the right driver.
// store.Get().Init("mysql", "root@tcp(127.0.0.1:3306)/test", "instructions")
// store.Get().Init("cassone", "crudtest", "instructions")
// store.Get().Init("192.168.59.103:27017", "crudtest", "instructions")
// store.Get().Init("192.168.59.103:28015", "test", "instructions")
search.Get().Init("memsearch")
indexer.Register("Post", SimpleIndexer{})
indexer.Register("Like", SimpleIndexer{})
indexer.Register("Comment", SimpleIndexer{})
indexer.Run(c, 2)
defer indexer.WaitForDone(c)
log.Debug("Store initialized. Checking search...")
uid := newUser()
// Let's get started. User 'uid' creates a new Post.
// This Post shares a url, adds some text and some tags.
tags := [3]string{"search", "cat", "videos"}
err = store.NewUpdate("User", uid).SetSource(uid).AddChild("Post").
Set("url", "www.google.com").Set("body", "You can search for cat videos here").
Set("tags", tags).Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
// Now let's add a comment and two likes to our new post.
// One user would add a comment and one like. Another user would
// just like the post.
//
// It's best to have the same 'source' for one set of operations.
// In REST APIs, this is how things would always be. Each REST call
// is from one user (and never two different users).
// This way the creation of like "entity", and the properties
// of that new like entity have the same source.
//
// So, here's Step 1: A new user would add a comment, and like the post.
fmt.Print("Added a new post by user")
user := printAndGetUser(uid)
post := user.Post[0]
p := store.NewUpdate("Post", post.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
p.AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on the post", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
// Step 2: Another user would now like the post.
p = store.NewUpdate("Post", post.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added a Comment and 2 Likes on Post")
user = printAndGetUser(uid)
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment := post.Comment[0]
// Now another user likes and replies to the comment that was added above.
// So, it's a comment within a comment.
p = store.NewUpdate("Comment", comment.Id).SetSource(newUser())
p.AddChild("Like").Set("thumb", 1)
p.AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on comment", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added a Comment and a Like on Comment")
user = printAndGetUser(uid)
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment = post.Comment[0]
if len(comment.Like) == 0 {
log.Fatalf("No like found: %+v", comment)
}
like := comment.Like[0]
// So far we have this structure:
// User
// L Post
// L 2 * Like
// L Comment
// L Comment
// L Like
// This is what most social platforms do. But, let's go
// one level further, and also comment on the Likes on Comment.
// User
// L Post
// L 2 * Like
// L Comment
// L Comment
// L Like
// L Comment
// Another user Comments on the Like on Comment on Post.
p = store.NewUpdate("Like", like.Id).SetSource(newUser()).
AddChild("Comment").Set("body",
fmt.Sprintf("Comment %s on Like", x.UniqueString(2)))
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Added Comment on Like")
user = printAndGetUser(uid)
{
docs, err := search.Get().NewQuery("Like").Order("data.source").Run()
if err != nil {
x.LogErr(log, err).Fatal("While searching for Post")
return
}
for _, doc := range docs {
log.WithField("doc", doc).Debug("Resulting doc")
}
log.Debug("Search query over")
}
post = user.Post[0]
if len(post.Comment) == 0 {
log.Fatalf("No comment found: %+v", post)
}
comment = post.Comment[0]
p = store.NewUpdate("Comment", comment.Id).SetSource(newUser()).Set("censored", true)
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
q := store.NewQuery(comment.Id).UptoDepth(0)
result, err := q.Run()
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Set censored=true on comment")
prettyPrintResult(*result)
user = printAndGetUser(uid)
post = user.Post[0]
if pid, err := store.Parent(post.Id); err == nil {
if pid != user.Id {
log.Fatal("Post's parent id doesn't match user id.")
return
}
log.WithFields(logrus.Fields{
"id": post.Id,
"parent_id": pid,
"user_id": user.Id,
}).Debug("Parent id matches")
} else {
log.Fatal(err.Error())
return
}
if len(post.Like) == 0 {
log.Fatalf("No like found: %+v", post)
}
like = post.Like[0]
p = store.NewUpdate("Like", like.Id).SetSource(newUser()).MarkDeleted()
err = p.Execute(c)
if err != nil {
log.Fatalf("Error: %v", err)
}
q = store.NewQuery(uid).Collect("Post")
q.Collect("Like").UptoDepth(10)
q.Collect("Comment").UptoDepth(10).FilterOut("censored")
result, err = q.Run()
if err != nil {
log.Fatalf("Error: %v", err)
}
fmt.Print("Filter out censored Comment and mark one Like as deleted.")
prettyPrintResult(*result)
// By now we have a fairly complex Post structure. CRUD for
// which would have been a lot of work to put together using
// typical SQL / NoSQL tables.
{
ch := make(chan x.Entity, 10)
done := make(chan bool)
go processChannel(ch, done)
num, last, err := store.Get().Iterate("", 100, ch)
if err != nil {
x.LogErr(log, err).Fatal("While iterating")
return
}
fmt.Printf("Found %d results\n", num)
fmt.Printf("Last Entity: %+v\n", last)
close(ch)
<-done
}
{
fmt.Println()
fmt.Println()
fmt.Print("Searching for doc with url = www.google.com")
q := search.Get().NewQuery("Post").Order("-data.activity")
q.NewAndFilter().AddExact("data.url", "www.google.com")
docs, err := q.Run()
if err != nil {
x.LogErr(log, err).Fatal("While searching for Post")
return
}
for _, doc := range docs {
js, err := json.MarshalIndent(doc, "", "' ")
if err != nil {
log.Fatalf("While marshal: %v\n", err)
return
}
fmt.Printf("\n%s\n%s\n%s\n\n", sep1, string(js), sep2)
// log.WithField("doc", doc).Debug("Resulting doc")
}
log.Debug("Search query over")
}
}