package indexer
import (
"sort"
"sync"
"github.com/aslanides/gocrud/req"
"github.com/aslanides/gocrud/search"
"github.com/aslanides/gocrud/x"
)
var log = x.Log("indexer")
// Indexer functions are called automatically by store operations.
// These functions are used to determine which entities need updating,
// and then re-generate their corresponding documents, which then get
// re-indexed into search engine, overwriting past
// (using versioning, if available) documents.
type Indexer interface {
// OnUpdate is called when an entity is updated due to a Commit
// on either itself, or it's direct children. Note that each
// child entity would also be called with OnUpdate. This function
// should return the Entity Ids, which need regeneration.
OnUpdate(x.Entity) []x.Entity
// Regenerate would be called on entities which need to be reprocessed
// due to a change. The workflow is:
// store.Commit -> search.OnUpdate -> Regenerate
Regenerate(x.Entity) x.Doc
}
package leveldb
import (
"errors"
"fmt"
"github.com/aslanides/gocrud/store"
"github.com/aslanides/gocrud/x"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/util"
)
var log = x.Log("leveldb")
type Leveldb struct {
db *leveldb.DB
opt *opt.Options
}
func (l *Leveldb) SetBloomFilter(bits int) {
l.opt = &opt.Options{
Filter: filter.NewBloomFilter(bits),
}
}
func (l *Leveldb) Init(args ...string) {
if len(args) != 1 {
log.WithField("args", args).Fatal("Invalid arguments")
return
package memsearch
import (
"errors"
"fmt"
"reflect"
"regexp"
"sort"
"strings"
"github.com/Sirupsen/logrus"
"github.com/aslanides/gocrud/search"
"github.com/aslanides/gocrud/x"
)
var log = x.Log("memsearch")
type MemSearch struct {
docs map[string]x.Doc
}
type MemQuery struct {
kind string
Docs []x.Doc
filter *MemFilter
filterType int // 0 = no filter, 1 = AND, 2 = OR
from int
limit int
order string
}
// Package store provides an interface for data store operations, to
// allow for easy extensibility to support various datastores. Also, provides
// the standardized Update and Query interfaces to data stores.
package store
import (
"github.com/aslanides/gocrud/x"
)
var log = x.Log("store")
// All the data CRUD operations are run via this Store interface.
// Implement this interface to add support for a datastore.
type Store interface {
// Init is used to initialize store driver.
Init(args ...string)
// Commit writes the array of instructions to the data store.
Commit(its []*x.Instruction) error
// IsNew returns true if the entity id provided doesn't exist in the
// store. Note that this entity id is never solely the row primary key,
// because multiple rows can (and most surely will) have the same entity id.
IsNew(entityId string) bool
// GetEntity retrieves all the rows for the given subject id, parses them
// into instructions, appends them to the array, and returns it. Any error
// encountered during these steps is also returned.
GetEntity(entityId string) ([]x.Instruction, error)
// Iterate allows for a way to page over all the entities stored in the table.
_ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
// _ "github.com/aslanides/gocrud/drivers/elasticsearch"
_ "github.com/aslanides/gocrud/drivers/leveldb"
_ "github.com/aslanides/gocrud/drivers/memsearch"
// _ "github.com/aslanides/gocrud/drivers/datastore"
// _ "github.com/aslanides/gocrud/drivers/sqlstore"
// _ "github.com/aslanides/gocrud/drivers/cassandra"
// _ "github.com/aslanides/gocrud/drivers/mongodb"
// _ "github.com/aslanides/gocrud/drivers/rethinkdb"
)
var debug = flag.Bool("debug", false, "Set debug level.")
var log = x.Log("social")
var c *req.Context
type Like struct {
Id string `json:"id,omitempty"`
}
type Comment struct {
Id string `json:"id,omitempty"`
Comment []Comment `json:"Comment,omitempty"`
Like []Like `json:"Like,omitempty"`
}
type Post struct {
Id string `json:"id,omitempty"`
Comment []Comment `json:"Comment,omitempty"`
// Package search provides a way to index entities and run relatively
// complicated search queries, best served outside of data stores, and
// by specialized search engines like ElasticSearch or Solr etc.
//
// This tackles the limitations caused by gocrud in terms of filtering
// and sort operations which would otherwise would need to be done at
// application level.
package search
import "github.com/aslanides/gocrud/x"
var log = x.Log("search")
// All the search operations are run via this Search interface.
// Implement this interface to add support for a search engine.
// Note that the term Entity is being used interchangeably with
// the term Subject. An Entity has a kind, and has an id.
// Query interface provides the search api encapsulator, responsible for
// generating the right query for the engine, and then running it.
type Query interface {
// NewAndFilter would return a filter which would run AND operation
// among individual filter queries.
NewAndFilter() FilterQuery
// NewOrFilter would return a filter which would run OR operation
// among individual filter queries.
NewOrFilter() FilterQuery
// From would set the offset from the first result. Use it along
// with Limit(int) to do pagination.
// Cassandra driver can now be imported, and initialized in social.go,
// or any other client.
// import _ "github.com/aslanides/gocrud/drivers/cassandra"
// Initialize in main():
// store.Get().Init("cassone", "crudtest", "instructions")
package cassandra
import (
"fmt"
"github.com/aslanides/gocrud/store"
"github.com/aslanides/gocrud/x"
"github.com/gocql/gocql"
)
var log = x.Log("cassandra")
type Cassandra struct {
session *gocql.Session
}
var kIsNew, kInsert, kSelect, kScan string
func (cs *Cassandra) SetSession(session *gocql.Session) {
cs.session = session
}
func (cs *Cassandra) setTableName(tablename string) {
kIsNew = fmt.Sprintf("select subject_id from %s where subject_id = ?", tablename)
kInsert = fmt.Sprintf(`insert into %s (ts, subject_id, subject_type, predicate,
object, object_id, nano_ts, source) values (now(), ?, ?, ?, ?, ?, ?, ?)`, tablename)
// If on Mac, find the IP address of the docker host
// $ boot2docker ip
// 192.168.59.103
// For linux it's 127.0.0.1.
// Now you can go to 192.168.59.103:8080 (or 127.0.0.1:8080) and create
// table 'instructions'. Once created, create an index by going to
// 'Data Explorer', and running this:
// r.db('test').table('instructions').indexCreate('SubjectId')
import (
"github.com/aslanides/gocrud/store"
"github.com/aslanides/gocrud/x"
r "github.com/dancannon/gorethink"
)
var log = x.Log("rethinkdb")
type RethinkDB struct {
session *r.Session
table string
}
func (rdb *RethinkDB) SetSession(session *r.Session) {
rdb.session = session
}
func (rdb *RethinkDB) Init(args ...string) {
if len(args) != 3 {
log.WithField("args", args).Fatal("Invalid arguments")
return
}
package sqlstore
import (
"database/sql"
"fmt"
"github.com/aslanides/gocrud/store"
"github.com/aslanides/gocrud/x"
)
var log = x.Log("sqlstore")
type Sql struct {
db *sql.DB
}
var sqlInsert *sql.Stmt
var sqlIsNew, sqlSelect string
func (s *Sql) Init(args ...string) {
if len(args) != 3 {
log.WithField("args", args).Fatal("Invalid arguments")
return
}
dbtype := args[0]
source := args[1]
tablename := args[2]
var err error
s.db, err = sql.Open(dbtype, source)
package elasticsearch
import (
"errors"
"reflect"
"github.com/aslanides/gocrud/search"
"github.com/aslanides/gocrud/x"
"gopkg.in/olivere/elastic.v2"
)
var log = x.Log("elasticsearch")
// Elastic encapsulates elastic search client, and implements methods declared
// by search.Engine.
type Elastic struct {
client *elastic.Client
}
// ElasticQuery implements methods declared by search.Query.
type ElasticQuery struct {
client *elastic.Client
sort string
from int
limit int
kind string
filter *ElasticFilter
filterType int // 0 = no filter, 1 = AND, 2 = OR
}
type ElasticFilter struct {
// This package is the initialization point for the api.
// In particular, in your init (/main) function, the flow
// is to create a req.Context and fill in required options.
// Setting table prefix, length for unique strings generated
// to assign to new entities, and setting the storage system.
package req
import "github.com/aslanides/gocrud/x"
var log = x.Log("req")
type Context struct {
NumCharsUnique int // 62^num unique strings
Updates chan x.Entity
HasIndexer bool
}
func NewContext(numChars int) *Context {
ctx := new(Context)
ctx.NumCharsUnique = numChars
ctx.HasIndexer = false
ctx.Updates = nil
return ctx
}
func NewContextWithUpdates(numChars, buffer int) *Context {
ctx := new(Context)
ctx.NumCharsUnique = numChars
ctx.Updates = make(chan x.Entity, buffer)
ctx.HasIndexer = true
return ctx
// To test this mongodb integration, run mongodb in docker
// docker run -d --name mongo -p 27017:27017 mongo:latest
// If on Mac, find the IP address of the docker host
// $ boot2docker ip
// 192.168.59.103
// For linux it's 127.0.0.1.
import (
"github.com/aslanides/gocrud/store"
"github.com/aslanides/gocrud/x"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
)
var log = x.Log("mongodb")
// MongoDB store backed by MongoDB
type MongoDB struct {
session *mgo.Session
database string
collection string
}
// Init setup a new collection using the name provided
func (mdb *MongoDB) Init(args ...string) {
if len(args) != 3 {
log.WithField("args", args).Fatal("Invalid arguments")
return
}