Ejemplo n.º 1
0
func TestVersions(t *testing.T) {
	path, err := ioutil.TempDir("", "gocrudldb_")
	if err != nil {
		t.Fatal("Opening leveldb file")
		return
	}
	store.Get().Init(path) // leveldb

	c := req.NewContext(10)
	var d int
	for d = 660; d < 670; d++ {
		if err = store.NewUpdate("Ticker", "GOOG").SetSource("nasdaq").
			Set("price", d).Execute(c); err != nil {
			t.Errorf("When updating store: %+v", err)
			t.Fail()
		}
	}
	result, err := store.NewQuery("GOOG").Run()
	if err != nil {
		t.Errorf("When querying store: %+v", err)
		t.Fail()
	}
	versions, present := result.Columns["price"]
	if !present {
		t.Errorf("Column price should be present: %+v", result)
		t.Fail()
	}
	if versions.Count() != 10 {
		t.Errorf("Num count expected: 10. Got: %v", versions.Count())
	}
	if versions.Latest().Value.(float64) != 669 {
		t.Errorf("Latest value expected 669. Got: %v", versions.Latest())
	}
	if versions.Oldest().Value.(float64) != 660 {
		t.Errorf("Oldest value expected 660. Got: %v", versions.Oldest())
	}

	type jval struct {
		Kind   string `json:"kind,omitempty"`
		Id     string `json:"id,omitempty"`
		Source string `json:"modifier,omitempty"`
		Price  int    `json:"price,omitempty"`
	}
	js, err := result.ToJson()
	if err != nil {
		t.Errorf("While converting to JSON: %v", err)
	}
	t.Log(string(js))
	var jv jval
	if err = json.Unmarshal(js, &jv); err != nil {
		t.Errorf("While unmarshal to struct: %v", err)
	}
	if jv.Price != 669 {
		t.Errorf("Latest value expected 669. Got: %+v", jv)
	}
	if jv.Source != "nasdaq" {
		t.Errorf("Source expected nasdaq. Got: %+v", jv)
	}
}
Ejemplo n.º 2
0
func ExampleServer() {
	store.Get().Init("/tmp/ldb_" + x.UniqueString(10))
	search.Get().Init("memsearch")
	indexer.Register("EntityKind", SimpleIndexer{})

	server := indexer.NewServer(100, 5)
	server.InfiniteLoop(30 * time.Minute)
	// This would never exit.
	// OR, you could also just run this once, if you're
	// testing your setup.
	server.LoopOnce()
	server.Finish() // Finish is only useful when you're looping once.
}
Ejemplo n.º 3
0
// LoopOnce would cycle over all entities in the store, and re-index them.
func (s *Server) LoopOnce() {
	var total uint64
	from := ""
	for {
		found, last, err := store.Get().Iterate(from, 1000, s.ch)
		if err != nil {
			x.LogErr(log, err).Error("While iterating")
			return
		}
		if found == 0 {
			log.WithField("total", total).Info("Reached end of cycle")
			return
		}
		log.WithFields(logrus.Fields{
			"num_processed": found,
			"last":          last,
		}).Debug("Iteration chunk done")
		total += uint64(found)
		from = last.Id
	}
	log.Fatal("This should never be reached.")
	return
}
Ejemplo n.º 4
0
func main() {
	rand.Seed(0) // Keep output consistent.
	flag.Parse()
	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
	} else {
		logrus.SetLevel(logrus.ErrorLevel)
	}

	c = req.NewContextWithUpdates(10, 1000) // 62^10 permutations

	// Initialize leveldb.
	dirname, err := ioutil.TempDir("", "ldb_")
	if err != nil {
		log.Fatalf("While creating temp directory: %v\n", err)
		return
	}
	defer os.RemoveAll(dirname)
	store.Get().Init(dirname)

	// Initialize Elasticsearch.
	// search.Get().Init("http://192.168.59.103:9200")

	// Other possible initializations. Remember to import the right driver.
	// store.Get().Init("mysql", "root@tcp(127.0.0.1:3306)/test", "instructions")
	// store.Get().Init("cassone", "crudtest", "instructions")
	// store.Get().Init("192.168.59.103:27017", "crudtest", "instructions")
	// store.Get().Init("192.168.59.103:28015", "test", "instructions")

	search.Get().Init("memsearch")
	indexer.Register("Post", SimpleIndexer{})
	indexer.Register("Like", SimpleIndexer{})
	indexer.Register("Comment", SimpleIndexer{})
	indexer.Run(c, 2)
	defer indexer.WaitForDone(c)

	log.Debug("Store initialized. Checking search...")
	uid := newUser()

	// Let's get started. User 'uid' creates a new Post.
	// This Post shares a url, adds some text and some tags.
	tags := [3]string{"search", "cat", "videos"}
	err = store.NewUpdate("User", uid).SetSource(uid).AddChild("Post").
		Set("url", "www.google.com").Set("body", "You can search for cat videos here").
		Set("tags", tags).Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	// Now let's add a comment and two likes to our new post.
	// One user would add a comment and one like. Another user would
	// just like the post.
	//
	// It's best to have the same 'source' for one set of operations.
	// In REST APIs, this is how things would always be. Each REST call
	// is from one user (and never two different users).
	// This way the creation of like "entity", and the properties
	// of that new like entity have the same source.
	//
	// So, here's Step 1: A new user would add a comment, and like the post.
	fmt.Print("Added a new post by user")
	user := printAndGetUser(uid)
	post := user.Post[0]

	p := store.NewUpdate("Post", post.Id).SetSource(newUser())
	p.AddChild("Like").Set("thumb", 1)
	p.AddChild("Comment").Set("body",
		fmt.Sprintf("Comment %s on the post", x.UniqueString(2)))
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	// Step 2: Another user would now like the post.
	p = store.NewUpdate("Post", post.Id).SetSource(newUser())
	p.AddChild("Like").Set("thumb", 1)
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Added a Comment and 2 Likes on Post")

	user = printAndGetUser(uid)
	post = user.Post[0]
	if len(post.Comment) == 0 {
		log.Fatalf("No comment found: %+v", post)
	}
	comment := post.Comment[0]

	// Now another user likes and replies to the comment that was added above.
	// So, it's a comment within a comment.
	p = store.NewUpdate("Comment", comment.Id).SetSource(newUser())
	p.AddChild("Like").Set("thumb", 1)
	p.AddChild("Comment").Set("body",
		fmt.Sprintf("Comment %s on comment", x.UniqueString(2)))
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Added a Comment and a Like on Comment")
	user = printAndGetUser(uid)
	post = user.Post[0]
	if len(post.Comment) == 0 {
		log.Fatalf("No comment found: %+v", post)
	}
	comment = post.Comment[0]
	if len(comment.Like) == 0 {
		log.Fatalf("No like found: %+v", comment)
	}
	like := comment.Like[0]

	// So far we have this structure:
	// User
	//  L Post
	//         L 2 * Like
	//         L Comment
	//            L Comment
	//            L Like

	// This is what most social platforms do. But, let's go
	// one level further, and also comment on the Likes on Comment.
	// User
	//    L Post
	//         L 2 * Like
	//         L Comment
	//            L Comment
	//            L Like
	//                 L Comment

	// Another user Comments on the Like on Comment on Post.

	p = store.NewUpdate("Like", like.Id).SetSource(newUser()).
		AddChild("Comment").Set("body",
		fmt.Sprintf("Comment %s on Like", x.UniqueString(2)))
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Added Comment on Like")
	user = printAndGetUser(uid)

	{
		docs, err := search.Get().NewQuery("Like").Order("data.source").Run()
		if err != nil {
			x.LogErr(log, err).Fatal("While searching for Post")
			return
		}
		for _, doc := range docs {
			log.WithField("doc", doc).Debug("Resulting doc")
		}
		log.Debug("Search query over")
	}

	post = user.Post[0]
	if len(post.Comment) == 0 {
		log.Fatalf("No comment found: %+v", post)
	}
	comment = post.Comment[0]
	p = store.NewUpdate("Comment", comment.Id).SetSource(newUser()).Set("censored", true)
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	q := store.NewQuery(comment.Id).UptoDepth(0)
	result, err := q.Run()
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Set censored=true on comment")
	prettyPrintResult(*result)

	user = printAndGetUser(uid)
	post = user.Post[0]
	if pid, err := store.Parent(post.Id); err == nil {
		if pid != user.Id {
			log.Fatal("Post's parent id doesn't match user id.")
			return
		}
		log.WithFields(logrus.Fields{
			"id":        post.Id,
			"parent_id": pid,
			"user_id":   user.Id,
		}).Debug("Parent id matches")
	} else {
		log.Fatal(err.Error())
		return
	}

	if len(post.Like) == 0 {
		log.Fatalf("No like found: %+v", post)
	}
	like = post.Like[0]
	p = store.NewUpdate("Like", like.Id).SetSource(newUser()).MarkDeleted()
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	q = store.NewQuery(uid).Collect("Post")
	q.Collect("Like").UptoDepth(10)
	q.Collect("Comment").UptoDepth(10).FilterOut("censored")
	result, err = q.Run()
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Filter out censored Comment and mark one Like as deleted.")
	prettyPrintResult(*result)
	// By now we have a fairly complex Post structure. CRUD for
	// which would have been a lot of work to put together using
	// typical SQL / NoSQL tables.

	{
		ch := make(chan x.Entity, 10)
		done := make(chan bool)
		go processChannel(ch, done)
		num, last, err := store.Get().Iterate("", 100, ch)
		if err != nil {
			x.LogErr(log, err).Fatal("While iterating")
			return
		}
		fmt.Printf("Found %d results\n", num)
		fmt.Printf("Last Entity: %+v\n", last)
		close(ch)
		<-done
	}

	{
		fmt.Println()
		fmt.Println()
		fmt.Print("Searching for doc with url = www.google.com")
		q := search.Get().NewQuery("Post").Order("-data.activity")
		q.NewAndFilter().AddExact("data.url", "www.google.com")
		docs, err := q.Run()
		if err != nil {
			x.LogErr(log, err).Fatal("While searching for Post")
			return
		}
		for _, doc := range docs {
			js, err := json.MarshalIndent(doc, "", "'  ")
			if err != nil {
				log.Fatalf("While marshal: %v\n", err)
				return
			}
			fmt.Printf("\n%s\n%s\n%s\n\n", sep1, string(js), sep2)
			// log.WithField("doc", doc).Debug("Resulting doc")
		}
		log.Debug("Search query over")
	}
}