Beispiel #1
0
package indexer

import (
	"sort"
	"sync"

	"github.com/aslanides/gocrud/req"
	"github.com/aslanides/gocrud/search"
	"github.com/aslanides/gocrud/x"
)

var log = x.Log("indexer")

// Indexer functions are called automatically by store operations.
// These functions are used to determine which entities need updating,
// and then re-generate their corresponding documents, which then get
// re-indexed into search engine, overwriting past
// (using versioning, if available) documents.
type Indexer interface {
	// OnUpdate is called when an entity is updated due to a Commit
	// on either itself, or it's direct children. Note that each
	// child entity would also be called with OnUpdate. This function
	// should return the Entity Ids, which need regeneration.
	OnUpdate(x.Entity) []x.Entity

	// Regenerate would be called on entities which need to be reprocessed
	// due to a change. The workflow is:
	// store.Commit -> search.OnUpdate -> Regenerate
	Regenerate(x.Entity) x.Doc
}
Beispiel #2
0
package leveldb

import (
	"errors"
	"fmt"

	"github.com/aslanides/gocrud/store"
	"github.com/aslanides/gocrud/x"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/filter"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/util"
)

var log = x.Log("leveldb")

type Leveldb struct {
	db  *leveldb.DB
	opt *opt.Options
}

func (l *Leveldb) SetBloomFilter(bits int) {
	l.opt = &opt.Options{
		Filter: filter.NewBloomFilter(bits),
	}
}

func (l *Leveldb) Init(args ...string) {
	if len(args) != 1 {
		log.WithField("args", args).Fatal("Invalid arguments")
		return
Beispiel #3
0
package memsearch

import (
	"errors"
	"fmt"
	"reflect"
	"regexp"
	"sort"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/aslanides/gocrud/search"
	"github.com/aslanides/gocrud/x"
)

var log = x.Log("memsearch")

type MemSearch struct {
	docs map[string]x.Doc
}

type MemQuery struct {
	kind       string
	Docs       []x.Doc
	filter     *MemFilter
	filterType int // 0 = no filter, 1 = AND, 2 = OR
	from       int
	limit      int
	order      string
}
Beispiel #4
0
// Package store provides an interface for data store operations, to
// allow for easy extensibility to support various datastores. Also, provides
// the standardized Update and Query interfaces to data stores.
package store

import (
	"github.com/aslanides/gocrud/x"
)

var log = x.Log("store")

// All the data CRUD operations are run via this Store interface.
// Implement this interface to add support for a datastore.
type Store interface {
	// Init is used to initialize store driver.
	Init(args ...string)

	// Commit writes the array of instructions to the data store.
	Commit(its []*x.Instruction) error

	// IsNew returns true if the entity id provided doesn't exist in the
	// store. Note that this entity id is never solely the row primary key,
	// because multiple rows can (and most surely will) have the same entity id.
	IsNew(entityId string) bool

	// GetEntity retrieves all the rows for the given subject id, parses them
	// into instructions, appends them to the array, and returns it. Any error
	// encountered during these steps is also returned.
	GetEntity(entityId string) ([]x.Instruction, error)

	// Iterate allows for a way to page over all the entities stored in the table.
Beispiel #5
0
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"

	// _ "github.com/aslanides/gocrud/drivers/elasticsearch"
	_ "github.com/aslanides/gocrud/drivers/leveldb"
	_ "github.com/aslanides/gocrud/drivers/memsearch"
	// _ "github.com/aslanides/gocrud/drivers/datastore"
	// _ "github.com/aslanides/gocrud/drivers/sqlstore"
	// _ "github.com/aslanides/gocrud/drivers/cassandra"
	// _ "github.com/aslanides/gocrud/drivers/mongodb"
	// _ "github.com/aslanides/gocrud/drivers/rethinkdb"
)

var debug = flag.Bool("debug", false, "Set debug level.")

var log = x.Log("social")
var c *req.Context

type Like struct {
	Id string `json:"id,omitempty"`
}

type Comment struct {
	Id      string    `json:"id,omitempty"`
	Comment []Comment `json:"Comment,omitempty"`
	Like    []Like    `json:"Like,omitempty"`
}

type Post struct {
	Id      string    `json:"id,omitempty"`
	Comment []Comment `json:"Comment,omitempty"`
Beispiel #6
0
// Package search provides a way to index entities and run relatively
// complicated search queries, best served outside of data stores, and
// by specialized search engines like ElasticSearch or Solr etc.
//
// This tackles the limitations caused by gocrud in terms of filtering
// and sort operations which would otherwise would need to be done at
// application level.
package search

import "github.com/aslanides/gocrud/x"

var log = x.Log("search")

// All the search operations are run via this Search interface.
// Implement this interface to add support for a search engine.
// Note that the term Entity is being used interchangeably with
// the term Subject. An Entity has a kind, and has an id.

// Query interface provides the search api encapsulator, responsible for
// generating the right query for the engine, and then running it.
type Query interface {
	// NewAndFilter would return a filter which would run AND operation
	// among individual filter queries.
	NewAndFilter() FilterQuery

	// NewOrFilter would return a filter which would run OR operation
	// among individual filter queries.
	NewOrFilter() FilterQuery

	// From would set the offset from the first result. Use it along
	// with Limit(int) to do pagination.
Beispiel #7
0
// Cassandra driver can now be imported, and initialized in social.go,
// or any other client.
// import _ "github.com/aslanides/gocrud/drivers/cassandra"
// Initialize in main():
// store.Get().Init("cassone", "crudtest", "instructions")
package cassandra

import (
	"fmt"

	"github.com/aslanides/gocrud/store"
	"github.com/aslanides/gocrud/x"
	"github.com/gocql/gocql"
)

var log = x.Log("cassandra")

type Cassandra struct {
	session *gocql.Session
}

var kIsNew, kInsert, kSelect, kScan string

func (cs *Cassandra) SetSession(session *gocql.Session) {
	cs.session = session
}

func (cs *Cassandra) setTableName(tablename string) {
	kIsNew = fmt.Sprintf("select subject_id from %s where subject_id = ?", tablename)
	kInsert = fmt.Sprintf(`insert into %s (ts, subject_id, subject_type, predicate,
	object, object_id, nano_ts, source) values (now(), ?, ?, ?, ?, ?, ?, ?)`, tablename)
Beispiel #8
0
// If on Mac, find the IP address of the docker host
// $ boot2docker ip
// 192.168.59.103
// For linux it's 127.0.0.1.
// Now you can go to 192.168.59.103:8080 (or 127.0.0.1:8080) and create
// table 'instructions'. Once created, create an index by going to
// 'Data Explorer', and running this:
// r.db('test').table('instructions').indexCreate('SubjectId')

import (
	"github.com/aslanides/gocrud/store"
	"github.com/aslanides/gocrud/x"
	r "github.com/dancannon/gorethink"
)

var log = x.Log("rethinkdb")

type RethinkDB struct {
	session *r.Session
	table   string
}

func (rdb *RethinkDB) SetSession(session *r.Session) {
	rdb.session = session
}

func (rdb *RethinkDB) Init(args ...string) {
	if len(args) != 3 {
		log.WithField("args", args).Fatal("Invalid arguments")
		return
	}
Beispiel #9
0
package sqlstore

import (
	"database/sql"
	"fmt"

	"github.com/aslanides/gocrud/store"
	"github.com/aslanides/gocrud/x"
)

var log = x.Log("sqlstore")

type Sql struct {
	db *sql.DB
}

var sqlInsert *sql.Stmt
var sqlIsNew, sqlSelect string

func (s *Sql) Init(args ...string) {
	if len(args) != 3 {
		log.WithField("args", args).Fatal("Invalid arguments")
		return
	}

	dbtype := args[0]
	source := args[1]
	tablename := args[2]

	var err error
	s.db, err = sql.Open(dbtype, source)
Beispiel #10
0
package elasticsearch

import (
	"errors"
	"reflect"

	"github.com/aslanides/gocrud/search"
	"github.com/aslanides/gocrud/x"
	"gopkg.in/olivere/elastic.v2"
)

var log = x.Log("elasticsearch")

// Elastic encapsulates elastic search client, and implements methods declared
// by search.Engine.
type Elastic struct {
	client *elastic.Client
}

// ElasticQuery implements methods declared by search.Query.
type ElasticQuery struct {
	client     *elastic.Client
	sort       string
	from       int
	limit      int
	kind       string
	filter     *ElasticFilter
	filterType int // 0 = no filter, 1 = AND, 2 = OR
}

type ElasticFilter struct {
Beispiel #11
0
// This package is the initialization point for the api.
// In particular, in your init (/main) function, the flow
// is to create a req.Context and fill in required options.
// Setting table prefix, length for unique strings generated
// to assign to new entities, and setting the storage system.
package req

import "github.com/aslanides/gocrud/x"

var log = x.Log("req")

type Context struct {
	NumCharsUnique int // 62^num unique strings
	Updates        chan x.Entity
	HasIndexer     bool
}

func NewContext(numChars int) *Context {
	ctx := new(Context)
	ctx.NumCharsUnique = numChars
	ctx.HasIndexer = false
	ctx.Updates = nil
	return ctx
}

func NewContextWithUpdates(numChars, buffer int) *Context {
	ctx := new(Context)
	ctx.NumCharsUnique = numChars
	ctx.Updates = make(chan x.Entity, buffer)
	ctx.HasIndexer = true
	return ctx
Beispiel #12
0
// To test this mongodb integration, run mongodb in docker
// docker run -d --name mongo -p 27017:27017 mongo:latest
// If on Mac, find the IP address of the docker host
// $ boot2docker ip
// 192.168.59.103
// For linux it's 127.0.0.1.

import (
	"github.com/aslanides/gocrud/store"
	"github.com/aslanides/gocrud/x"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

var log = x.Log("mongodb")

// MongoDB store backed by MongoDB
type MongoDB struct {
	session    *mgo.Session
	database   string
	collection string
}

// Init setup a new collection using the name provided
func (mdb *MongoDB) Init(args ...string) {
	if len(args) != 3 {
		log.WithField("args", args).Fatal("Invalid arguments")
		return
	}