Exemplo n.º 1
0
func ExampleServer() {
	store.Get().Init("/tmp/ldb_" + x.UniqueString(10))
	search.Get().Init("memsearch")
	indexer.Register("EntityKind", SimpleIndexer{})

	server := indexer.NewServer(100, 5)
	server.InfiniteLoop(30 * time.Minute)
	// This would never exit.
	// OR, you could also just run this once, if you're
	// testing your setup.
	server.LoopOnce()
	server.Finish() // Finish is only useful when you're looping once.
}
Exemplo n.º 2
0
func AddDocs(e search.Engine) {
	for idx, name := range galaxies {
		var d x.Doc
		d.Id = x.UniqueString(5)
		d.Kind = "Galaxy"
		d.NanoTs = time.Now().UnixNano()
		m := make(map[string]interface{})
		m["name"] = name
		m["pos"] = idx
		d.Data = m

		if err := e.Update(d); err != nil {
			log.Fatalf("While updating: %v", err)
			return
		}
	}
}
Exemplo n.º 3
0
func (l *Leveldb) Commit(_ string, its []*x.Instruction) error {
	var keys []string
	for _, it := range its {
		var key string
		for m := 0; m < 10; m++ {
			key = fmt.Sprintf("%s_%s", it.SubjectId, x.UniqueString(5))
			log.WithField("key", key).Debug("Checking existence of key")
			if has, err := l.db.Has([]byte(key), nil); err != nil {
				x.LogErr(log, err).WithField("key", key).Error("While check if key exists")
				continue
			} else if has {
				continue
			} else {
				break
			}
			log.Errorf("Exhausted %d tries", m)
			return errors.New("Exhausted tries")
		}
		log.WithField("key", key).Debug("Is unique")
		keys = append(keys, key)
	}

	b := new(leveldb.Batch)
	for idx, it := range its {
		key := []byte(keys[idx])
		buf, err := it.GobEncode()
		if err != nil {
			x.LogErr(log, err).Error("While encoding")
			return err
		}
		b.Put(key, buf)
	}
	if err := l.db.Write(b, nil); err != nil {
		x.LogErr(log, err).Error("While writing to db")
		return err
	}
	log.Debugf("%d instructions committed", len(its))

	return nil
}
Exemplo n.º 4
0
func main() {
	rand.Seed(time.Now().UnixNano())
	fmt.Println("Running...")

	ctx = new(req.Context)

	l := new(store.Leveldb)
	l.SetBloomFilter(13)
	ctx.Store = l
	ctx.Store.Init("leveldb", "test")

	e := echo.New()
	e.SetDebug(true)
	e.Use(mw.Recover())
	e.Static("/assets/", "public/assets")
	e.Get("/webms/:name", func(c *echo.Context) error {
		resp := c.Response()
		name := c.Param("name")
		resp.Header().Add("Content-Type", "video/webm")
		http.ServeFile(resp, c.Request(), "webms/"+name)
		return nil
	})
	e.Static("/webms/", "webms")

	//e.ServeFile("/", "public/index.html")
	e.Get("/", func(c *echo.Context) error {
		http.ServeFile(c.Response(), c.Request(), "public/index.html")
		return nil
	})

	e.ServeFile("/upload", "public/upload.html")
	e.Post("/upload", func(c *echo.Context) error {
		req := c.Request()

		name := req.FormValue("name")
		ttags := req.FormValue("tags")
		tags := strings.Split(ttags, " ")
		path := x.UniqueString(10)

		// Read files
		file := req.MultipartForm.File["file"]
		src, err := file[0].Open()
		if err != nil {
			return err
		}
		defer src.Close()

		// Destination file
		dst, err := os.Create("webms/" + path + ".webm")
		if err != nil {
			return err
		}
		defer dst.Close()

		if _, err = io.Copy(dst, src); err != nil {
			return err
		}

		if err = api.Get("WebmCont", rootid).SetSource(rootid).AddChild("Webm").Set("path", path).
			Set("tags", tags).Set("name", name).Execute(ctx); err != nil {
			return err
		}

		return c.String(http.StatusOK, "eyo uploaded file")
	})

	e.Get("/webm", func(c *echo.Context) error {
		result, err := api.NewQuery("WebmCont", rootid).Collect("Webm").Run(ctx)
		if err != nil {
			return err
		}
		by, err := result.ToJson()
		if err != nil {
			return err
		}
		return c.String(200, string(by))
	})

	e.Get("/webm/filter/:filter", func(c *echo.Context) error {
		tags := strings.Split(c.Param("filter"), "+")
		result, err := api.NewQuery("WebmCont", rootid).Collect("Webm").Run(ctx)
		if err != nil {
			return err
		}
		by, err := filter(result, tags...)
		if err != nil {
			return err
		}
		return c.String(200, string(by))
	})

	e.Get("/webm/:id", func(c *echo.Context) error {
		result, err := api.NewQuery("Webm", c.Param("id")).Run(ctx)
		if err != nil {
			return err
		}
		by, err := result.ToJson()
		if err != nil {
			return err
		}
		return c.String(200, string(by))
	})

	e.Get("/webm/:id/tag/:tag", func(c *echo.Context) error {
		result, err := api.NewQuery("Webm", c.Param("id")).Run(ctx)
		if err != nil {
			return err
		}
		var tags []interface{}
		tagb, ok := result.Columns["tags"].Value.([]interface{})
		if ok {
			tags = tagb
		}
		cont := contains(tags, c.Param("tag"))
		var atags []string
		var resp string
		if cont {
			resp = "removed"
			atags = convertButLeave(tags, c.Param("tag"))
		} else {
			resp = "added"
			atags = convert(tags)
			atags = append(atags, c.Param("tag"))
		}
		err = api.Get("Webm", c.Param("id")).SetSource(rootid).Set("tags", atags).Execute(ctx)
		if err != nil {
			return err
		}

		return c.String(200, resp)
	})

	graceful.ListenAndServe(e.Server(":8080"), 5*time.Second)
}
Exemplo n.º 5
0
func main() {
	rand.Seed(time.Now().UnixNano())
	fmt.Println("Running...")
	flag.Parse()

	c = new(req.Context)

	if *storeType == "leveldb" {
		l := new(store.Leveldb)
		l.SetBloomFilter(13)
		c.Store = l
		c.Store.Init(*storeType, "/tmp/ldb_"+x.UniqueString(10))

	} else if *storeType == "cass" {
		cluster := gocql.NewCluster("192.168.59.103")
		cluster.Keyspace = "crudtest"
		cluster.Consistency = gocql.Quorum
		cass := new(store.Cassandra)

		if session, err := cluster.CreateSession(); err != nil {
			panic(err)
		} else {
			cass.SetSession(session)
		}
		c.Store = cass
		c.Store.Init(*storeType, "instructions")

	} else if *storeType == "mysql" {
		db, err := sql.Open("mysql", "root@tcp(127.0.0.1:3306)/test")
		if err != nil {
			panic(err)
		}
		if err = db.Ping(); err != nil {
			panic(err)
		}
		log.Info("Connection to mysql successful")
		sqldb := new(store.Sql)
		sqldb.SetDb(db)
		c.Store = sqldb
		c.Store.Init(*storeType, "instructions")

	} else if *storeType == "postgres" {
		db, err := sql.Open("postgres", "postgres://localhost/test?sslmode=disable")
		if err != nil {
			panic(err)
		}
		if err = db.Ping(); err != nil {
			panic(err)
		}
		log.Info("Connection to postgres successful")
		sqldb := new(store.Sql)
		sqldb.SetDb(db)
		c.Store = sqldb
		c.Store.Init(*storeType, "instructions")

	} else if *storeType == "rethinkdb" {
		session, err := r.Connect(r.ConnectOpts{
			// Address:  "192.168.59.103:28015",
			Address:  "localhost:28015",
			Database: "crudtest",
		})
		if err != nil {
			panic(err)
		}
		rethinkdb := new(store.RethinkDB)
		rethinkdb.SetSession(session)
		c.Store = rethinkdb
		c.Store.Init(*storeType, "instructions")

	} else if *storeType == "mongodb" {
		// "192.168.59.103:27017"
		session, err := mgo.Dial("localhost:27017")
		if err != nil {
			panic(err)
		}
		session.SetMode(mgo.Monotonic, true)
		mongodb := new(store.MongoDB)
		mongodb.SetSession(session, "crudtest")
		c.Store = mongodb
		c.Store.Init(*storeType, "instructions")

	} else if *storeType == "datastore" {
		c.TablePrefix = "Test-"
		c.Store = new(store.Datastore)
		c.Store.Init(*storeType, "gce-project-id")

	} else {
		panic("Invalid store")
	}

	var err error
	uid := newUser()

	// Let's get started. User 'uid' creates a new Post.
	// This Post shares a url, adds some text and some tags.
	tags := [3]string{"search", "cat", "videos"}
	err = api.Get("User", uid).SetSource(uid).AddChild("Post").
		Set("url", "www.google.com").Set("body", "You can search for cat videos here").
		Set("tags", tags).Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Stored Post")

	// Now let's add a comment and two likes to our new post.
	// One user would add a comment and one like. Another user would
	// just like the post.
	//
	// It's best to have the same 'source' for one set of operations.
	// In REST APIs, this is how things would always be. Each REST call
	// is from one user (and never two different users).
	// This way the creation of like "entity", and the properties
	// of that new like entity have the same source.
	//
	// So, here's Step 1: A new user would add a comment, and like the post.
	user := printAndGetUser(uid)
	post := user.Post[0]

	p := api.Get("Post", post.Id).SetSource(newUser())
	p.AddChild("Like").Set("thumb", 1)
	p.AddChild("Comment").Set("body",
		fmt.Sprintf("Comment %s on the post", x.UniqueString(2)))
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	// Step 2: Another user would now like the post.
	p = api.Get("Post", post.Id).SetSource(newUser())
	p.AddChild("Like").Set("thumb", 1)
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Added 1 Comment and 2 Like on Post")

	user = printAndGetUser(uid)
	post = user.Post[0]
	if len(post.Comment) == 0 {
		log.Fatalf("No comment found: %+v", post)
	}
	comment := post.Comment[0]

	// Now another user likes and replies to the comment that was added above.
	// So, it's a comment within a comment.
	p = api.Get("Comment", comment.Id).SetSource(newUser())
	p.AddChild("Like").Set("thumb", 1)
	p.AddChild("Comment").Set("body",
		fmt.Sprintf("Comment %s on comment", x.UniqueString(2)))
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Added Comment on Comment")
	user = printAndGetUser(uid)
	post = user.Post[0]
	if len(post.Comment) == 0 {
		log.Fatalf("No comment found: %+v", post)
	}
	comment = post.Comment[0]
	if len(comment.Like) == 0 {
		log.Fatalf("No like found: %+v", comment)
	}
	like := comment.Like[0]

	// So far we have this structure:
	// User
	//  L Post
	//         L 2 * Like
	//         L Comment
	//            L Comment
	//            L Like

	// This is what most social platforms do. But, let's go
	// one level further, and also comment on the Likes on Comment.
	// User
	//    L Post
	//         L 2 * Like
	//         L Comment
	//            L Comment
	//            L Like
	//                 L Comment

	// Another user Comments on the Like on Comment on Post.

	p = api.Get("Like", like.Id).SetSource(newUser()).
		AddChild("Comment").Set("body",
		fmt.Sprintf("Comment %s on Like", x.UniqueString(2)))
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Added Comment on Like")
	user = printAndGetUser(uid)

	post = user.Post[0]
	if len(post.Comment) == 0 {
		log.Fatalf("No comment found: %+v", post)
	}
	comment = post.Comment[0]
	p = api.Get("Comment", comment.Id).SetSource(newUser()).Set("censored", true)
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	q := api.NewQuery("Comment", comment.Id).UptoDepth(0)
	result, err := q.Run(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	js, err := result.ToJson()
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Printf("\n%s\n%s\n%s\n", sep, string(js), sep)
	user = printAndGetUser(uid)

	post = user.Post[0]
	if len(post.Like) == 0 {
		log.Fatalf("No like found: %+v", post)
	}
	like = post.Like[0]
	p = api.Get("Like", like.Id).SetSource(newUser()).MarkDeleted()
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	q = api.NewQuery("User", uid).Collect("Post")
	q.Collect("Like").UptoDepth(10)
	q.Collect("Comment").UptoDepth(10).FilterOut("censored")
	result, err = q.Run(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	js, err = result.ToJson()
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Printf("\n%s\n%s\n%s\n", sep, string(js), sep)
	// By now we have a fairly complex Post structure. CRUD for
	// which would have been a lot of work to put together using
	// typical SQL / NoSQL tables.
}
Exemplo n.º 6
0
func newUser() string {
	return "uid_" + x.UniqueString(3)
}
Exemplo n.º 7
0
func (n *Update) doExecute(c *req.Context, its *[]*x.Instruction) error {
	for pred, val := range n.edges {
		if len(n.source) == 0 {
			return errors.New(fmt.Sprintf(
				"No source specified for id: %v kind: %v", n.id, n.kind))
		}

		i := new(x.Instruction)
		i.SubjectId = n.id
		i.SubjectType = n.kind
		i.Predicate = pred

		if b, err := json.Marshal(val); err != nil {
			return err
		} else {
			i.Object = b
		}
		i.Source = n.source
		i.NanoTs = n.NanoTs
		log.WithField("instruction", i).Debug("Pushing to list")
		*its = append(*its, i)
	}

	if len(n.children) == 0 {
		return nil
	}
	if len(n.source) == 0 {
		return errors.New(fmt.Sprintf(
			"No source specified for id: %v kind: %v", n.id, n.kind))
	}

	// Children can only be added, not deleted via API. But they can be stopped
	// from being retrieved.
	// Scenario: How do I stop childA from being retrieved?
	// Answer:
	// Modify child by adding a 'deleted' edge
	// Get(ChildKind, ChildId).Set("deleted", true).Execute(c)
	//
	// Then for retrieval from parent:
	// NewQuery(ParentKind, ParentId).Collect(ChildKind).FilterOut("deleted")
	// This would remove all children with a 'deleted' edge.
	// Better still
	// Get(ChildKind, ChildId).MarkDeleted().Execute(c)
	// would automatically filter out that child and it's children
	// from being retrieved.

	for _, child := range n.children {
		if len(child.id) > 0 {
			log.WithField("child_id", child.id).Fatal(
				"Child id should be empty for all current use cases")
			return errors.New("Non empty child id")
		}

		for idx := 0; ; idx++ { // Retry loop.
			child.id = x.UniqueString(c.NumCharsUnique)
			log.WithField("id", child.id).Debug("Checking availability of new id")
			if isnew := Get().IsNew(child.id); isnew {
				log.WithField("id", child.id).Debug("New id available")
				break
			}
			if idx >= 30 {
				return errors.New("Unable to find new id")
			}
		}
		// Create edge from parent to child
		i := new(x.Instruction)
		i.SubjectId = n.id
		i.SubjectType = n.kind
		i.Predicate = child.kind
		i.ObjectId = child.id
		i.Source = n.source
		i.NanoTs = n.NanoTs
		log.WithField("instruction", i).Debug("Pushing to list")
		*its = append(*its, i)

		// Create edge from child to parent
		i = new(x.Instruction)
		i.SubjectId = child.id
		i.SubjectType = child.kind
		i.Predicate = "_parent_"
		i.ObjectId = n.id
		i.Source = n.source
		i.NanoTs = n.NanoTs
		log.WithField("instruction", i).Debug("Pushing to list")
		*its = append(*its, i)

		if err := child.doExecute(c, its); err != nil {
			return err
		}
	}
	return nil
}
Exemplo n.º 8
0
func ExampleUniqueString() {
	u := x.UniqueString(3)
	fmt.Println(len(u))
	// Output: 3
}
Exemplo n.º 9
0
func main() {
	rand.Seed(0) // Keep output consistent.
	flag.Parse()
	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
	} else {
		logrus.SetLevel(logrus.ErrorLevel)
	}

	c = req.NewContextWithUpdates(10, 1000) // 62^10 permutations

	// Initialize leveldb.
	dirname, err := ioutil.TempDir("", "ldb_")
	if err != nil {
		log.Fatalf("While creating temp directory: %v\n", err)
		return
	}
	defer os.RemoveAll(dirname)
	store.Get().Init(dirname)

	// Initialize Elasticsearch.
	// search.Get().Init("http://192.168.59.103:9200")

	// Other possible initializations. Remember to import the right driver.
	// store.Get().Init("mysql", "root@tcp(127.0.0.1:3306)/test", "instructions")
	// store.Get().Init("cassone", "crudtest", "instructions")
	// store.Get().Init("192.168.59.103:27017", "crudtest", "instructions")
	// store.Get().Init("192.168.59.103:28015", "test", "instructions")

	search.Get().Init("memsearch")
	indexer.Register("Post", SimpleIndexer{})
	indexer.Register("Like", SimpleIndexer{})
	indexer.Register("Comment", SimpleIndexer{})
	indexer.Run(c, 2)
	defer indexer.WaitForDone(c)

	log.Debug("Store initialized. Checking search...")
	uid := newUser()

	// Let's get started. User 'uid' creates a new Post.
	// This Post shares a url, adds some text and some tags.
	tags := [3]string{"search", "cat", "videos"}
	err = store.NewUpdate("User", uid).SetSource(uid).AddChild("Post").
		Set("url", "www.google.com").Set("body", "You can search for cat videos here").
		Set("tags", tags).Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	// Now let's add a comment and two likes to our new post.
	// One user would add a comment and one like. Another user would
	// just like the post.
	//
	// It's best to have the same 'source' for one set of operations.
	// In REST APIs, this is how things would always be. Each REST call
	// is from one user (and never two different users).
	// This way the creation of like "entity", and the properties
	// of that new like entity have the same source.
	//
	// So, here's Step 1: A new user would add a comment, and like the post.
	fmt.Print("Added a new post by user")
	user := printAndGetUser(uid)
	post := user.Post[0]

	p := store.NewUpdate("Post", post.Id).SetSource(newUser())
	p.AddChild("Like").Set("thumb", 1)
	p.AddChild("Comment").Set("body",
		fmt.Sprintf("Comment %s on the post", x.UniqueString(2)))
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	// Step 2: Another user would now like the post.
	p = store.NewUpdate("Post", post.Id).SetSource(newUser())
	p.AddChild("Like").Set("thumb", 1)
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Added a Comment and 2 Likes on Post")

	user = printAndGetUser(uid)
	post = user.Post[0]
	if len(post.Comment) == 0 {
		log.Fatalf("No comment found: %+v", post)
	}
	comment := post.Comment[0]

	// Now another user likes and replies to the comment that was added above.
	// So, it's a comment within a comment.
	p = store.NewUpdate("Comment", comment.Id).SetSource(newUser())
	p.AddChild("Like").Set("thumb", 1)
	p.AddChild("Comment").Set("body",
		fmt.Sprintf("Comment %s on comment", x.UniqueString(2)))
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Added a Comment and a Like on Comment")
	user = printAndGetUser(uid)
	post = user.Post[0]
	if len(post.Comment) == 0 {
		log.Fatalf("No comment found: %+v", post)
	}
	comment = post.Comment[0]
	if len(comment.Like) == 0 {
		log.Fatalf("No like found: %+v", comment)
	}
	like := comment.Like[0]

	// So far we have this structure:
	// User
	//  L Post
	//         L 2 * Like
	//         L Comment
	//            L Comment
	//            L Like

	// This is what most social platforms do. But, let's go
	// one level further, and also comment on the Likes on Comment.
	// User
	//    L Post
	//         L 2 * Like
	//         L Comment
	//            L Comment
	//            L Like
	//                 L Comment

	// Another user Comments on the Like on Comment on Post.

	p = store.NewUpdate("Like", like.Id).SetSource(newUser()).
		AddChild("Comment").Set("body",
		fmt.Sprintf("Comment %s on Like", x.UniqueString(2)))
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Added Comment on Like")
	user = printAndGetUser(uid)

	{
		docs, err := search.Get().NewQuery("Like").Order("data.source").Run()
		if err != nil {
			x.LogErr(log, err).Fatal("While searching for Post")
			return
		}
		for _, doc := range docs {
			log.WithField("doc", doc).Debug("Resulting doc")
		}
		log.Debug("Search query over")
	}

	post = user.Post[0]
	if len(post.Comment) == 0 {
		log.Fatalf("No comment found: %+v", post)
	}
	comment = post.Comment[0]
	p = store.NewUpdate("Comment", comment.Id).SetSource(newUser()).Set("censored", true)
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	q := store.NewQuery(comment.Id).UptoDepth(0)
	result, err := q.Run()
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Set censored=true on comment")
	prettyPrintResult(*result)

	user = printAndGetUser(uid)
	post = user.Post[0]
	if pid, err := store.Parent(post.Id); err == nil {
		if pid != user.Id {
			log.Fatal("Post's parent id doesn't match user id.")
			return
		}
		log.WithFields(logrus.Fields{
			"id":        post.Id,
			"parent_id": pid,
			"user_id":   user.Id,
		}).Debug("Parent id matches")
	} else {
		log.Fatal(err.Error())
		return
	}

	if len(post.Like) == 0 {
		log.Fatalf("No like found: %+v", post)
	}
	like = post.Like[0]
	p = store.NewUpdate("Like", like.Id).SetSource(newUser()).MarkDeleted()
	err = p.Execute(c)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	q = store.NewQuery(uid).Collect("Post")
	q.Collect("Like").UptoDepth(10)
	q.Collect("Comment").UptoDepth(10).FilterOut("censored")
	result, err = q.Run()
	if err != nil {
		log.Fatalf("Error: %v", err)
	}
	fmt.Print("Filter out censored Comment and mark one Like as deleted.")
	prettyPrintResult(*result)
	// By now we have a fairly complex Post structure. CRUD for
	// which would have been a lot of work to put together using
	// typical SQL / NoSQL tables.

	{
		ch := make(chan x.Entity, 10)
		done := make(chan bool)
		go processChannel(ch, done)
		num, last, err := store.Get().Iterate("", 100, ch)
		if err != nil {
			x.LogErr(log, err).Fatal("While iterating")
			return
		}
		fmt.Printf("Found %d results\n", num)
		fmt.Printf("Last Entity: %+v\n", last)
		close(ch)
		<-done
	}

	{
		fmt.Println()
		fmt.Println()
		fmt.Print("Searching for doc with url = www.google.com")
		q := search.Get().NewQuery("Post").Order("-data.activity")
		q.NewAndFilter().AddExact("data.url", "www.google.com")
		docs, err := q.Run()
		if err != nil {
			x.LogErr(log, err).Fatal("While searching for Post")
			return
		}
		for _, doc := range docs {
			js, err := json.MarshalIndent(doc, "", "'  ")
			if err != nil {
				log.Fatalf("While marshal: %v\n", err)
				return
			}
			fmt.Printf("\n%s\n%s\n%s\n\n", sep1, string(js), sep2)
			// log.WithField("doc", doc).Debug("Resulting doc")
		}
		log.Debug("Search query over")
	}
}