func TestVersions(t *testing.T) { path, err := ioutil.TempDir("", "gocrudldb_") if err != nil { t.Fatal("Opening leveldb file") return } store.Get().Init(path) // leveldb c := req.NewContext(10) var d int for d = 660; d < 670; d++ { if err = store.NewUpdate("Ticker", "GOOG").SetSource("nasdaq"). Set("price", d).Execute(c); err != nil { t.Errorf("When updating store: %+v", err) t.Fail() } } result, err := store.NewQuery("GOOG").Run() if err != nil { t.Errorf("When querying store: %+v", err) t.Fail() } versions, present := result.Columns["price"] if !present { t.Errorf("Column price should be present: %+v", result) t.Fail() } if versions.Count() != 10 { t.Errorf("Num count expected: 10. Got: %v", versions.Count()) } if versions.Latest().Value.(float64) != 669 { t.Errorf("Latest value expected 669. Got: %v", versions.Latest()) } if versions.Oldest().Value.(float64) != 660 { t.Errorf("Oldest value expected 660. Got: %v", versions.Oldest()) } type jval struct { Kind string `json:"kind,omitempty"` Id string `json:"id,omitempty"` Source string `json:"modifier,omitempty"` Price int `json:"price,omitempty"` } js, err := result.ToJson() if err != nil { t.Errorf("While converting to JSON: %v", err) } t.Log(string(js)) var jv jval if err = json.Unmarshal(js, &jv); err != nil { t.Errorf("While unmarshal to struct: %v", err) } if jv.Price != 669 { t.Errorf("Latest value expected 669. Got: %+v", jv) } if jv.Source != "nasdaq" { t.Errorf("Source expected nasdaq. Got: %+v", jv) } }
func ExampleServer() { store.Get().Init("/tmp/ldb_" + x.UniqueString(10)) search.Get().Init("memsearch") indexer.Register("EntityKind", SimpleIndexer{}) server := indexer.NewServer(100, 5) server.InfiniteLoop(30 * time.Minute) // This would never exit. // OR, you could also just run this once, if you're // testing your setup. server.LoopOnce() server.Finish() // Finish is only useful when you're looping once. }
func ExampleSearch() { path, err := ioutil.TempDir("", "gocrudldb_") if err != nil { x.LogErr(log, err).Fatal("Opening file") return } store.Get().Init(path) // leveldb search.Get().Init() // memsearch // Run indexer to update entities in search engine in real time. c := req.NewContextWithUpdates(10, 100) indexer.Register("Child", SimpleIndexer{}) indexer.Run(c, 2) u := store.NewUpdate("Root", "bigbang").SetSource("author") for i := 0; i < 10; i++ { child := u.AddChild("Child").Set("pos", i).Set("particle", particles[i]) if i == 5 { child.MarkDeleted() // This shouldn't be retrieved anymore. } } if err = u.Execute(c); err != nil { x.LogErr(log, err).Fatal("While updating") return } indexer.WaitForDone(c) // Block until indexing is done. docs, err := search.Get().NewQuery("Child").Order("-data.pos").Run() if err != nil { x.LogErr(log, err).Fatal("While searching") return } fmt.Println("docs:", len(docs)) for _, doc := range docs { m := doc.Data.(map[string]interface{}) fmt.Println(m["pos"], m["particle"]) } // Output: // docs: 9 // 9 higgs boson // 8 boson // 7 photon // 6 bottom // 4 down // 3 gluon // 2 top // 1 charm // 0 up }
func ExampleStore() { path, err := ioutil.TempDir("", "gocrudldb_") if err != nil { x.LogErr(log, err).Fatal("Opening file") return } store.Get().Init(path) // leveldb // Update some data. c := req.NewContext(10) // 62^10 permutations err = store.NewUpdate("Root", "bigbang").SetSource("author"). Set("when", "13.8 billion years ago").Set("explosive", true).Execute(c) if err != nil { x.LogErr(log, err).Fatal("Commiting update") return } // Retrieve that data result, err := store.NewQuery("bigbang").Run() if err != nil { x.LogErr(log, err).Fatal("While querying store") return } fmt.Println(result.Kind) // Root fmt.Println(result.Id) // bigbang data := result.ToMap() { val, ok := data["explosive"] if !ok { log.Fatal("creator should be set") return } fmt.Println(val) // true } { val, ok := data["when"] if !ok { log.Fatal("creator should be set") return } fmt.Println(val) } // Output: // Root // bigbang // true // 13.8 billion years ago }
// LoopOnce would cycle over all entities in the store, and re-index them. func (s *Server) LoopOnce() { var total uint64 from := "" for { found, last, err := store.Get().Iterate(from, 1000, s.ch) if err != nil { x.LogErr(log, err).Error("While iterating") return } if found == 0 { log.WithField("total", total).Info("Reached end of cycle") return } log.WithFields(logrus.Fields{ "num_processed": found, "last": last, }).Debug("Iteration chunk done") total += uint64(found) from = last.Id } log.Fatal("This should never be reached.") return }
func main() { rand.Seed(0) // Keep output consistent. flag.Parse() if *debug { logrus.SetLevel(logrus.DebugLevel) } else { logrus.SetLevel(logrus.ErrorLevel) } c = req.NewContextWithUpdates(10, 1000) // 62^10 permutations // Initialize leveldb. dirname, err := ioutil.TempDir("", "ldb_") if err != nil { log.Fatalf("While creating temp directory: %v\n", err) return } defer os.RemoveAll(dirname) store.Get().Init(dirname) // Initialize Elasticsearch. // search.Get().Init("http://192.168.59.103:9200") // Other possible initializations. Remember to import the right driver. // store.Get().Init("mysql", "root@tcp(127.0.0.1:3306)/test", "instructions") // store.Get().Init("cassone", "crudtest", "instructions") // store.Get().Init("192.168.59.103:27017", "crudtest", "instructions") // store.Get().Init("192.168.59.103:28015", "test", "instructions") search.Get().Init("memsearch") indexer.Register("Post", SimpleIndexer{}) indexer.Register("Like", SimpleIndexer{}) indexer.Register("Comment", SimpleIndexer{}) indexer.Run(c, 2) defer indexer.WaitForDone(c) log.Debug("Store initialized. Checking search...") uid := newUser() // Let's get started. User 'uid' creates a new Post. // This Post shares a url, adds some text and some tags. tags := [3]string{"search", "cat", "videos"} err = store.NewUpdate("User", uid).SetSource(uid).AddChild("Post"). Set("url", "www.google.com").Set("body", "You can search for cat videos here"). Set("tags", tags).Execute(c) if err != nil { log.Fatalf("Error: %v", err) } // Now let's add a comment and two likes to our new post. // One user would add a comment and one like. Another user would // just like the post. // // It's best to have the same 'source' for one set of operations. // In REST APIs, this is how things would always be. Each REST call // is from one user (and never two different users). // This way the creation of like "entity", and the properties // of that new like entity have the same source. // // So, here's Step 1: A new user would add a comment, and like the post. fmt.Print("Added a new post by user") user := printAndGetUser(uid) post := user.Post[0] p := store.NewUpdate("Post", post.Id).SetSource(newUser()) p.AddChild("Like").Set("thumb", 1) p.AddChild("Comment").Set("body", fmt.Sprintf("Comment %s on the post", x.UniqueString(2))) err = p.Execute(c) if err != nil { log.Fatalf("Error: %v", err) } // Step 2: Another user would now like the post. p = store.NewUpdate("Post", post.Id).SetSource(newUser()) p.AddChild("Like").Set("thumb", 1) err = p.Execute(c) if err != nil { log.Fatalf("Error: %v", err) } fmt.Print("Added a Comment and 2 Likes on Post") user = printAndGetUser(uid) post = user.Post[0] if len(post.Comment) == 0 { log.Fatalf("No comment found: %+v", post) } comment := post.Comment[0] // Now another user likes and replies to the comment that was added above. // So, it's a comment within a comment. p = store.NewUpdate("Comment", comment.Id).SetSource(newUser()) p.AddChild("Like").Set("thumb", 1) p.AddChild("Comment").Set("body", fmt.Sprintf("Comment %s on comment", x.UniqueString(2))) err = p.Execute(c) if err != nil { log.Fatalf("Error: %v", err) } fmt.Print("Added a Comment and a Like on Comment") user = printAndGetUser(uid) post = user.Post[0] if len(post.Comment) == 0 { log.Fatalf("No comment found: %+v", post) } comment = post.Comment[0] if len(comment.Like) == 0 { log.Fatalf("No like found: %+v", comment) } like := comment.Like[0] // So far we have this structure: // User // L Post // L 2 * Like // L Comment // L Comment // L Like // This is what most social platforms do. But, let's go // one level further, and also comment on the Likes on Comment. // User // L Post // L 2 * Like // L Comment // L Comment // L Like // L Comment // Another user Comments on the Like on Comment on Post. p = store.NewUpdate("Like", like.Id).SetSource(newUser()). AddChild("Comment").Set("body", fmt.Sprintf("Comment %s on Like", x.UniqueString(2))) err = p.Execute(c) if err != nil { log.Fatalf("Error: %v", err) } fmt.Print("Added Comment on Like") user = printAndGetUser(uid) { docs, err := search.Get().NewQuery("Like").Order("data.source").Run() if err != nil { x.LogErr(log, err).Fatal("While searching for Post") return } for _, doc := range docs { log.WithField("doc", doc).Debug("Resulting doc") } log.Debug("Search query over") } post = user.Post[0] if len(post.Comment) == 0 { log.Fatalf("No comment found: %+v", post) } comment = post.Comment[0] p = store.NewUpdate("Comment", comment.Id).SetSource(newUser()).Set("censored", true) err = p.Execute(c) if err != nil { log.Fatalf("Error: %v", err) } q := store.NewQuery(comment.Id).UptoDepth(0) result, err := q.Run() if err != nil { log.Fatalf("Error: %v", err) } fmt.Print("Set censored=true on comment") prettyPrintResult(*result) user = printAndGetUser(uid) post = user.Post[0] if pid, err := store.Parent(post.Id); err == nil { if pid != user.Id { log.Fatal("Post's parent id doesn't match user id.") return } log.WithFields(logrus.Fields{ "id": post.Id, "parent_id": pid, "user_id": user.Id, }).Debug("Parent id matches") } else { log.Fatal(err.Error()) return } if len(post.Like) == 0 { log.Fatalf("No like found: %+v", post) } like = post.Like[0] p = store.NewUpdate("Like", like.Id).SetSource(newUser()).MarkDeleted() err = p.Execute(c) if err != nil { log.Fatalf("Error: %v", err) } q = store.NewQuery(uid).Collect("Post") q.Collect("Like").UptoDepth(10) q.Collect("Comment").UptoDepth(10).FilterOut("censored") result, err = q.Run() if err != nil { log.Fatalf("Error: %v", err) } fmt.Print("Filter out censored Comment and mark one Like as deleted.") prettyPrintResult(*result) // By now we have a fairly complex Post structure. CRUD for // which would have been a lot of work to put together using // typical SQL / NoSQL tables. { ch := make(chan x.Entity, 10) done := make(chan bool) go processChannel(ch, done) num, last, err := store.Get().Iterate("", 100, ch) if err != nil { x.LogErr(log, err).Fatal("While iterating") return } fmt.Printf("Found %d results\n", num) fmt.Printf("Last Entity: %+v\n", last) close(ch) <-done } { fmt.Println() fmt.Println() fmt.Print("Searching for doc with url = www.google.com") q := search.Get().NewQuery("Post").Order("-data.activity") q.NewAndFilter().AddExact("data.url", "www.google.com") docs, err := q.Run() if err != nil { x.LogErr(log, err).Fatal("While searching for Post") return } for _, doc := range docs { js, err := json.MarshalIndent(doc, "", "' ") if err != nil { log.Fatalf("While marshal: %v\n", err) return } fmt.Printf("\n%s\n%s\n%s\n\n", sep1, string(js), sep2) // log.WithField("doc", doc).Debug("Resulting doc") } log.Debug("Search query over") } }