func GetDB() *db.DB {
myDB, err := db.OpenDB(tiedotmartini2.DATABASE_DIR)
if err != nil {
panic(err)
}
return myDB
}
func Start(dir string, port int, tlsCrt, tlsKey, jwtPubKey, jwtPrivateKey string) {
var err error
HttpDB, err = db.OpenDB(dir)
if err != nil {
panic(err)
}
// These endpoints are always available and do not require JWT auth
http.HandleFunc("/", Welcome)
http.HandleFunc("/version", Version)
http.HandleFunc("/memstats", MemStats)
if jwtPrivateKey != "" {
// JWT support
ServeJWTEnabledEndpoints(jwtPubKey, jwtPrivateKey)
} else {
// No JWT
ServeEndpoints()
}
if tlsCrt != "" {
tdlog.Noticef("Will listen on all interfaces (HTTPS), port %d.", port)
if err := http.ListenAndServeTLS(fmt.Sprintf(":%d", port), tlsCrt, tlsKey, nil); err != nil {
tdlog.Panicf("Failed to start HTTPS service - %s", err)
}
} else {
tdlog.Noticef("Will listen on all interfaces (HTTP), port %d.", port)
http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
}
}
func initialize() {
// remove temp files
os.RemoveAll(DBdir)
// open db
d, err := db.OpenDB(DBdir)
if err != nil {
panic(err)
}
// create collection
if err := d.Create("Entries"); err != nil {
panic(err)
}
// collection instance
docEntries := d.Use("Entries")
// dummy data
entries := Entries{
&Entry{1, time.Now(), "Entry 1", "First Entry!"},
&Entry{2, time.Now(), "Golang", "Go language is awesome"},
}
// insert each
for _, entry := range entries {
docEntries.Insert(structs.New(entry).Map())
}
}
// Setup the database connection
func initDB() *db.Col {
// (Create if not exist) open a database
myDB, err := db.OpenDB("database")
if err != nil {
log.Println("Opening existing database")
} else {
log.Println("Creating new database")
}
// Create two collections: Feeds and Votes
if err := myDB.Create("Users"); err != nil {
log.Println("Using existing users database")
} else {
log.Println("Using fresh new users database")
}
users := myDB.Use("Users")
if err := users.Index([]string{"Email"}); err != nil {
log.Println("Using existing email index on users")
} else {
log.Println("Creating new index on email field of users database")
}
// Add a random user or 2
// users.Insert(dbRow{"Email": "*****@*****.**", "Password": "******"})
return users
}
func main() {
// It is very important to initialize random number generator seed!
rand.Seed(time.Now().UTC().UnixNano())
jsondb, err := db.OpenDB(dbname)
if err != nil {
panic(err)
}
defer jsondb.Close()
if _, err = os.Stat(filepath.Join(dbname, clname)); os.IsNotExist(err) {
if err = jsondb.Create(clname); err != nil {
panic(err)
}
/*
userscl = jsondb.Use(clname)
if err = userscl.Index([]string{"courseCode"}); err != nil {
panic(err)
}
*/
}
if _, err = os.Stat(filepath.Join(dbname, qlname)); os.IsNotExist(err) {
if err = jsondb.Create(qlname); err != nil {
panic(err)
}
qlCl = jsondb.Use(qlname)
if err := qlCl.Index([]string{"courseId"}); err != nil {
panic(err)
}
}
userscl = jsondb.Use(clname)
qlCl = jsondb.Use(qlname)
/*
needs to create index upon first time
if err := qlCl.Index([]string{"courseId"}); err != nil {
panic(err)
}
if err := userscl.Index([]string{"courseCode"}); err != nil {
panic(err)
}
*/
m := martini.Classic()
m.Get("/users", GetUsers)
m.Post("/users", CreateUser)
m.Get("/users/:id", GetUser)
m.Delete("/users/:id", DeleteUser)
m.Put("/users/:id", UpdateUser)
m.Get("/courses", GetCourses)
m.Get("/courses/:id", SearchCourse)
m.Post("/courses", CreateCourse)
m.Post("/questionlists", CreateQuestionList)
m.Get("/questionLists", SearchQuestionListByCourseId)
m.Get("/questionLists/:id", SearchQuestionList)
m.Post("/questionLists/:id", UpdateQuestionList)
m.Run()
}
func main() {
// It is very important to initialize random number generator seed!
rand.Seed(time.Now().UTC().UnixNano())
jsondb, err := db.OpenDB(dbname)
if err != nil {
panic(err)
}
defer jsondb.Close()
if _, err = os.Stat(filepath.Join(dbname, clname)); os.IsNotExist(err) {
if err = jsondb.Create(clname); err != nil {
panic(err)
}
}
userscl = jsondb.Use(clname)
userscl.Insert(map[string]interface{}{"name": "Course1", "status": "activ", "Age": 3})
userscl.Insert(map[string]interface{}{"name": "Course 2", "status": "active", "Age": 3})
userscl.Insert(map[string]interface{}{"name": "Course 3", "status": "golang.org", "Age": 3})
m := martini.Classic()
m.Get("/users", GetUsers)
m.Post("/users", CreateUser)
m.Get("/users/:id", GetUser)
m.Delete("/users/:id", DeleteUser)
m.Put("/users/:id", UpdateUser)
m.Get("/courses", SearchCourse)
m.Run()
}
// Create a temporary database and collection for benchmark use.
func mkTmpDBAndCol(dbPath string, colName string) (*db.DB, *db.Col) {
os.RemoveAll(dbPath)
tmpDB, err := db.OpenDB(dbPath)
if err != nil {
panic(err)
}
if err = tmpDB.Create(colName); err != nil {
panic(err)
}
return tmpDB, tmpDB.Use(colName)
}
func NewDatabaseConnection(rootPath string) (database *db.DB) {
dir := rootPath + "/data"
database, err := db.OpenDB(dir)
if err != nil {
panic(err)
}
initDatabase(database)
return
}
// Create a temporary database and collection for benchmark use.
func mkTmpDBAndCol(dbPath string, colName string) (col *db.Col) {
os.RemoveAll(dbPath)
tmpDB, err := db.OpenDB(dbPath)
if err != nil {
panic(err)
}
tmpCol, err := db.OpenCol(tmpDB, colName)
if err != nil {
panic(err)
}
return tmpCol
}
func (tdb *DBTiedot) OpenDatabase() {
dir := "/tmp/seed-db"
db, err := tiedot.OpenDB(dir)
if err != nil {
panic(err)
}
if err := db.Create("Users", 50); err != nil {
fmt.Println("Collection Users already created.")
}
tdb.Db = db
}
func (tdb *DBTiedot) OpenDB(constr string) (interface{}, error) {
dir := "/tmp/seed-db"
db, err := tiedot.OpenDB(dir)
if err != nil {
return nil, err
}
//if err := db.Create("Users"); err != nil {
// fmt.Println("Collection Users already created.")
//}
tdb.Db = db
return db, nil
}
func NewDatabase(path string) (*Database, error) {
d, err := db.OpenDB(path)
if err != nil {
return nil, err
}
d.Create("Projects")
d.Create("Resources")
return &Database{
db: d,
}, nil
}
func GetDB(path string) *DataStore {
db, err := db.OpenDB(path)
if err != nil {
panic(err)
}
db.Create("Users", 2)
users := db.Use("Users")
users.Index([]string{"nick", "ip"})
return &DataStore{
url: path,
db: db,
users: users,
}
}
func home(w http.ResponseWriter, req *http.Request) {
// local vars
var query interface{}
entries := Entries{}
queryResults := make(map[int]struct{})
// open database
d, err := db.OpenDB(DBdir)
if err != nil {
panic(err)
}
// use collection
docEntries := d.Use("Entries")
// build query from json and convert to interface{}
json.Unmarshal([]byte(`"all"`), &query)
// execute query and pass results to queryResults
if err := db.EvalQuery(query, docEntries, &queryResults); err != nil {
panic(err)
}
// queryResults contains []int of IDs
for id := range queryResults {
entry := Entry{}
readBack, _ := docEntries.Read(id)
// map[string]interface{} TO struct hack
j, _ := json.Marshal(readBack) // struct to json
json.Unmarshal(j, &entry) // json to actual type
entries = append(entries, &entry)
}
// compile template with data
if err := index.Execute(w, entries); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
func NewTieDb(dir string, maxDay int) *TieDb {
mydb, err := db.OpenDB(dir)
if err != nil {
panic(err)
}
tdb := &TieDb{
tdb: mydb,
RequestTable: newTable(mydb, "req"),
ResponseTable: newTable(mydb, "res"),
}
if maxDay > 0 {
maxTime := time.Now().Unix() - int64(maxDay)*86400
if tdb.RequestTable.Gc(maxTime) > 100 {
tdb.RequestTable.Scrub()
}
if tdb.ResponseTable.Gc(maxTime) > 100 {
tdb.ResponseTable.Scrub()
}
}
return tdb
}
func setupDb(pathDb string, eraseDataBase bool) {
myDBDir := pathDb
if eraseDataBase {
os.RemoveAll(myDBDir)
defer os.RemoveAll(myDBDir)
}
myDB, err := db.OpenDB(myDBDir)
if err != nil {
panic(err)
}
if eraseDataBase {
if err := myDB.Create("todo"); err != nil {
panic(err)
}
}
todoCol = myDB.Use("todo")
}
func New(dbDir string) (*ProxyStorage, error) {
myDb, err := db.OpenDB(dbDir)
if err != nil {
return nil, err
}
proxyCol := myDb.Use("proxy")
if proxyCol == nil {
if err := myDb.Create("proxy"); err != nil {
return nil, err
}
proxyCol = myDb.Use("proxy")
if err := proxyCol.Index([]string{"Ip"}); err != nil {
return nil, err
}
if err := proxyCol.Index([]string{"Port"}); err != nil {
return nil, err
}
}
logger.Info("Established db connection")
return &ProxyStorage{myDb, proxyCol}, nil
}
func main() {
myDB, err := db.OpenDB("data")
if err != nil {
panic(err)
}
if err := myDB.Create("Pages"); err != nil {
fmt.Println(err)
}
pageCol = myDB.Use("Pages")
http.HandleFunc("/commit/", CommitHandler)
http.HandleFunc("/e/", EditPageHandler)
http.Handle("/", http.FileServer(http.Dir("./public")))
http.HandleFunc("/gen", GeneratorHandler)
http.HandleFunc("/p/", PageHandler)
http.HandleFunc("/new", NewPageHandler)
http.ListenAndServe(":3001", nil)
}
func add(w http.ResponseWriter, req *http.Request) {
// make sure it's post
if req.Method != "POST" {
http.NotFound(w, req)
return
}
entry := Entry{
Name: req.FormValue("name"),
Message: req.FormValue("message"),
}
d, err := db.OpenDB(DBdir)
if err != nil {
panic(err)
}
docEntries := d.Use("Entries")
docEntries.Insert(structs.New(entry).Map())
http.Redirect(w, req, "/", http.StatusTemporaryRedirect)
}
// Create a new LevelDBEngine with the given file and options
func NewTiedotEngine(directory string, collections []string, dropPref DropPreference) *TiedotEngine {
db, err := tiedot.OpenDB(directory)
log.FatalIfErr(err, "Failure opening tiedot basedir err:")
for _, c := range collections {
if _, ok := db.StrCol[c]; ok {
log.V(4).Info("Collection %s already exists")
if dropPref == DropIfExist {
log.Info("Dropping collection %s due to dropIfExist option")
err = db.Drop(c)
log.FatalIfErr(err, "Failure dropping collection with name:%s err:", c)
err = db.Create(c, 1) // partition DB for use by up to 1 goroutines at a time
log.FatalIfErr(err, "Failure creating collection with name:%s err:", c)
}
} else {
log.V(4).Info("Creating collection %s")
err = db.Create(c, 1) // partition DB for use by up to 1 goroutines at a time
log.FatalIfErr(err, "Failure creating collection with name:%s err:", c)
}
}
tde := &TiedotEngine{
tiedot: db,
}
return tde
}
func GetDb() (myDB *db.DB, err error) {
myDB, err = db.OpenDB("/data/pastebin")
return
}
func PersistentSample1() {
myDBDir := "/Users/gianluca/Projects/golang/src/github.com/gi4nks/mosto/tmp/MyDatabase"
os.RemoveAll(myDBDir)
defer os.RemoveAll(myDBDir)
// (Create if not exist) open a database
myDB, err := db.OpenDB(myDBDir)
if err != nil {
panic(err)
}
// Create two collections: Commands and Votes
if err := myDB.Create("Commands"); err != nil {
tracer.Warning(err.Error())
}
// What collections do I now have?
for _, name := range myDB.AllCols() {
fmt.Printf("I have a collection called %s\n", name)
}
// ****************** Document Management ******************
// Start using a collection (the reference is valid until DB schema changes or Scrub is carried out)
commands := myDB.Use("Commands")
u4, err := uuid.NewV4()
if err != nil {
fmt.Println("error:", err)
return
}
tracer.News("1 - " + u4.String())
cmd1 := Command{Id: u4.String(), Name: "cmd1", Started: time.Now()}
u4, err = uuid.NewV4()
if err != nil {
fmt.Println("error:", err)
return
}
tracer.News("2 - " + u4.String())
cmd2 := Command{Id: u4.String(), Name: "cmd2", Started: time.Now()}
tracer.News(AsJson(cmd1))
tracer.News(AsJson(cmd2))
// Insert document (afterwards the docID uniquely identifies the document and will never change)
docID, err := commands.Insert(structs.Map(cmd1))
if err != nil {
panic(err)
}
// Read document
readBack, err := commands.Read(docID)
if err != nil {
panic(err)
}
fmt.Println("Document", docID, "is", readBack)
// Insert document (afterwards the docID uniquely identifies the document and will never change)
docID, err = commands.Insert(structs.Map(cmd2))
if err != nil {
panic(err)
}
// Read document
readBack, err = commands.Read(docID)
if err != nil {
panic(err)
}
fmt.Println("Document", docID, "is", readBack)
// Process all documents (note that document order is undetermined)
commands.ForEachDoc(func(id int, docContent []byte) (willMoveOn bool) {
fmt.Println("Document", id, "is", string(docContent))
return true // move on to the next document OR
return false // do not move on to the next document
})
// Gracefully close database
if err := myDB.Close(); err != nil {
panic(err)
}
}
func main() {
rand.Seed(time.Now().UTC().UnixNano())
var err error
var defaultMaxprocs int
if defaultMaxprocs, err = strconv.Atoi(os.Getenv("GOMAXPROCS")); err != nil {
defaultMaxprocs = runtime.NumCPU() * 2
}
// Parse CLI parameters
var mode, dir string
var port, maxprocs, benchSize int
var profile bool
flag.StringVar(&mode, "mode", "", "http|bench|bench2|bench3|example]")
flag.StringVar(&dir, "dir", "", "database directory")
flag.IntVar(&port, "port", 0, "listening port number")
flag.IntVar(&maxprocs, "gomaxprocs", defaultMaxprocs, "GOMAXPROCS")
flag.IntVar(&benchSize, "benchsize", 400000, "Benchmark sample size")
flag.BoolVar(&profile, "profile", false, "write profiler results to prof.out")
flag.BoolVar(&tdlog.VerboseLog, "verbose", true, "verbose logging true/false (default is true)")
flag.Parse()
// User must specify a mode to run
if mode == "" {
flag.PrintDefaults()
return
}
// Setup appropriate GOMAXPROCS parameter
runtime.GOMAXPROCS(maxprocs)
tdlog.Printf("GOMAXPROCS is set to %d", maxprocs)
if maxprocs < runtime.NumCPU() {
tdlog.Printf("GOMAXPROCS (%d) is less than number of CPUs (%d), this may affect performance. You can change it via environment variable GOMAXPROCS or by passing CLI parameter -gomaxprocs", maxprocs, runtime.NumCPU())
}
// Start profiler if enabled
if profile {
resultFile, err := os.Create("perf.out")
if err != nil {
tdlog.Panicf("Cannot create profiler result file %s", resultFile)
}
pprof.StartCPUProfile(resultFile)
defer pprof.StopCPUProfile()
}
switch mode {
case "http": // Run HTTP service (API V3)
if dir == "" {
tdlog.Fatal("Please specify database directory, for example -dir=/tmp/db")
}
if port == 0 {
tdlog.Fatal("Please specify port number, for example -port=8080")
}
db, err := db.OpenDB(dir)
if err != nil {
tdlog.Fatal(err)
}
v3.Start(db, port)
case "bench": // Benchmark scenarios
benchmark(benchSize)
case "bench2":
benchmark2(benchSize)
case "bench3":
benchmark3(benchSize)
case "example": // Embedded usage example
embeddedExample()
default:
flag.PrintDefaults()
return
}
}
func EmbeddedExample() {
// ****************** Collection Management ******************
myDBDir := "/Users/gianluca/Projects/golang/src/github.com/gi4nks/mosto/tmp/MyDatabase"
os.RemoveAll(myDBDir)
defer os.RemoveAll(myDBDir)
// (Create if not exist) open a database
myDB, err := db.OpenDB(myDBDir)
if err != nil {
panic(err)
}
// Create two collections: Feeds and Votes
if err := myDB.Create("Feeds"); err != nil {
panic(err)
}
if err := myDB.Create("Votes"); err != nil {
panic(err)
}
// What collections do I now have?
for _, name := range myDB.AllCols() {
fmt.Printf("I have a collection called %s\n", name)
}
// Rename collection "Votes" to "Points"
if err := myDB.Rename("Votes", "Points"); err != nil {
panic(err)
}
// Drop (delete) collection "Points"
if err := myDB.Drop("Points"); err != nil {
panic(err)
}
// Scrub (repair and compact) "Feeds"
if err := myDB.Scrub("Feeds"); err != nil {
panic(err)
}
// ****************** Document Management ******************
// Start using a collection (the reference is valid until DB schema changes or Scrub is carried out)
feeds := myDB.Use("Feeds")
// Insert document (afterwards the docID uniquely identifies the document and will never change)
docID, err := feeds.Insert(map[string]interface{}{
"name": "Go 1.2 is released",
"url": "golang.org"})
if err != nil {
panic(err)
}
// Read document
readBack, err := feeds.Read(docID)
if err != nil {
panic(err)
}
fmt.Println("Document", docID, "is", readBack)
// Update document
err = feeds.Update(docID, map[string]interface{}{
"name": "Go is very popular",
"url": "google.com"})
if err != nil {
panic(err)
}
// Process all documents (note that document order is undetermined)
feeds.ForEachDoc(func(id int, docContent []byte) (willMoveOn bool) {
fmt.Println("Document", id, "is", string(docContent))
return true // move on to the next document OR
return false // do not move on to the next document
})
// Delete document
if err := feeds.Delete(docID); err != nil {
panic(err)
}
// More complicated error handing - identify the error Type.
// In this example, the error code tells that the document no longer exists.
if err := feeds.Delete(docID); dberr.Type(err) == dberr.ErrorNoDoc {
fmt.Println("The document was already deleted")
}
// ****************** Index Management ******************
// Indexes assist in many types of queries
// Create index (path leads to document JSON attribute)
if err := feeds.Index([]string{"author", "name", "first_name"}); err != nil {
panic(err)
}
if err := feeds.Index([]string{"Title"}); err != nil {
panic(err)
}
if err := feeds.Index([]string{"Source"}); err != nil {
panic(err)
}
// What indexes do I have on collection A?
for _, path := range feeds.AllIndexes() {
fmt.Printf("I have an index on path %v\n", path)
}
// Remove index
if err := feeds.Unindex([]string{"author", "name", "first_name"}); err != nil {
panic(err)
}
// ****************** Queries ******************
// Prepare some documents for the query
feeds.Insert(map[string]interface{}{"Title": "New Go release", "Source": "golang.org", "Age": 3})
feeds.Insert(map[string]interface{}{"Title": "Kitkat is here", "Source": "google.com", "Age": 2})
feeds.Insert(map[string]interface{}{"Title": "Good Slackware", "Source": "slackware.com", "Age": 1})
/*
var query interface{}
json.Unmarshal([]byte(`[{"eq": "New Go release", "in": ["Title"]}, {"eq": "slackware.com", "in": ["Source"]}]`), &query)
queryResult := make(map[int]struct{}) // query result (document IDs) goes into map keys
if err := db.EvalQuery(query, feeds, &queryResult); err != nil {
panic(err)
}
// Query result are document IDs
for id := range queryResult {
// To get query result document, simply read it
readBack, err := feeds.Read(id)
if err != nil {
panic(err)
}
readBack = nil
}*/
// Gracefully close database
if err := myDB.Close(); err != nil {
panic(err)
}
}
// Start HTTP server and block until the server shuts down. Panic on error.
func Start(dir string, port int, tlsCrt, tlsKey, jwtPubKey, jwtPrivateKey, bind, authToken string) {
var err error
HttpDB, err = db.OpenDB(dir)
if err != nil {
panic(err)
}
// These endpoints are always available and do not require authentication
http.HandleFunc("/", Welcome)
http.HandleFunc("/version", Version)
http.HandleFunc("/memstats", MemStats)
// Install API endpoint handlers that may require authorization
var authWrap func(http.HandlerFunc) http.HandlerFunc
if authToken != "" {
tdlog.Noticef("API endpoints now require the pre-shared token in Authorization header.")
authWrap = func(originalHandler http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if "token "+authToken != r.Header.Get("Authorization") {
http.Error(w, "", http.StatusUnauthorized)
return
}
originalHandler(w, r)
}
}
} else if jwtPubKey != "" && jwtPrivateKey != "" {
tdlog.Noticef("API endpoints now require JWT in Authorization header.")
var publicKeyContent, privateKeyContent []byte
if publicKeyContent, err = ioutil.ReadFile(jwtPubKey); err != nil {
panic(err)
} else if publicKey, err = jwt.ParseRSAPublicKeyFromPEM(publicKeyContent); err != nil {
panic(err)
} else if privateKeyContent, err = ioutil.ReadFile(jwtPrivateKey); err != nil {
panic(err)
} else if privateKey, err = jwt.ParseRSAPrivateKeyFromPEM(privateKeyContent); err != nil {
panic(err)
}
jwtInitSetup()
authWrap = jwtWrap
// does not require JWT auth
http.HandleFunc("/getjwt", getJWT)
http.HandleFunc("/checkjwt", checkJWT)
} else {
tdlog.Noticef("API endpoints do not require Authorization header.")
authWrap = func(originalHandler http.HandlerFunc) http.HandlerFunc {
return originalHandler
}
}
// collection management (stop-the-world)
http.HandleFunc("/create", authWrap(Create))
http.HandleFunc("/rename", authWrap(Rename))
http.HandleFunc("/drop", authWrap(Drop))
http.HandleFunc("/all", authWrap(All))
http.HandleFunc("/scrub", authWrap(Scrub))
http.HandleFunc("/sync", authWrap(Sync))
// query
http.HandleFunc("/query", authWrap(Query))
http.HandleFunc("/count", authWrap(Count))
// document management
http.HandleFunc("/insert", authWrap(Insert))
http.HandleFunc("/get", authWrap(Get))
http.HandleFunc("/getpage", authWrap(GetPage))
http.HandleFunc("/update", authWrap(Update))
http.HandleFunc("/delete", authWrap(Delete))
http.HandleFunc("/approxdoccount", authWrap(ApproxDocCount))
// index management (stop-the-world)
http.HandleFunc("/index", authWrap(Index))
http.HandleFunc("/indexes", authWrap(Indexes))
http.HandleFunc("/unindex", authWrap(Unindex))
// misc (stop-the-world)
http.HandleFunc("/shutdown", authWrap(Shutdown))
http.HandleFunc("/dump", authWrap(Dump))
iface := "all interfaces"
if bind != "" {
iface = bind
}
if tlsCrt != "" {
tdlog.Noticef("Will listen on %s (HTTPS), port %d.", iface, port)
if err := http.ListenAndServeTLS(fmt.Sprintf("%s:%d", bind, port), tlsCrt, tlsKey, nil); err != nil {
tdlog.Panicf("Failed to start HTTPS service - %s", err)
}
} else {
tdlog.Noticef("Will listen on %s (HTTP), port %d.", iface, port)
http.ListenAndServe(fmt.Sprintf("%s:%d", bind, port), nil)
}
}
// example.go written in test case style
func TestAll(t *testing.T) {
myDBDir := "/tmp/tiedot_test_embeddedExample"
os.RemoveAll(myDBDir)
defer os.RemoveAll(myDBDir)
myDB, err := db.OpenDB(myDBDir)
if err != nil {
t.Fatal(err)
}
if err := myDB.Create("Feeds"); err != nil {
t.Fatal(err)
}
if err := myDB.Create("Votes"); err != nil {
t.Fatal(err)
}
for _, name := range myDB.AllCols() {
if name != "Feeds" && name != "Votes" {
t.Fatal(myDB.AllCols())
}
}
if err := myDB.Rename("Votes", "Points"); err != nil {
t.Fatal(err)
}
if err := myDB.Drop("Points"); err != nil {
t.Fatal(err)
}
if err := myDB.Scrub("Feeds"); err != nil {
t.Fatal(err)
}
feeds := myDB.Use("Feeds")
docID, err := feeds.Insert(map[string]interface{}{
"name": "Go 1.2 is released",
"url": "golang.org"})
if err != nil {
t.Fatal(err)
}
readBack, err := feeds.Read(docID)
if err != nil {
t.Fatal(err)
}
if !sameMap(readBack, map[string]interface{}{
"name": "Go 1.2 is released",
"url": "golang.org"}) {
t.Fatal(readBack)
}
err = feeds.Update(docID, map[string]interface{}{
"name": "Go is very popular",
"url": "google.com"})
if err != nil {
panic(err)
}
feeds.ForEachDoc(func(id int, docContent []byte) (willMoveOn bool) {
var doc map[string]interface{}
if json.Unmarshal(docContent, &doc) != nil {
t.Fatal("cannot deserialize")
}
if !sameMap(doc, map[string]interface{}{
"name": "Go is very popular",
"url": "google.com"}) {
t.Fatal(doc)
}
return true
})
if err := feeds.Delete(docID); err != nil {
t.Fatal(err)
}
if err := feeds.Delete(docID); dberr.Type(err) != dberr.ErrorNoDoc {
t.Fatal(err)
}
if err := feeds.Index([]string{"author", "name", "first_name"}); err != nil {
t.Fatal(err)
}
if err := feeds.Index([]string{"Title"}); err != nil {
t.Fatal(err)
}
if err := feeds.Index([]string{"Source"}); err != nil {
t.Fatal(err)
}
for _, path := range feeds.AllIndexes() {
joint := strings.Join(path, "")
if joint != "authornamefirst_name" && joint != "Title" && joint != "Source" {
t.Fatal(feeds.AllIndexes())
}
}
if err := feeds.Unindex([]string{"author", "name", "first_name"}); err != nil {
t.Fatal(err)
}
// ****************** Queries ******************
// Prepare some documents for the query
if _, err := feeds.Insert(map[string]interface{}{"Title": "New Go release", "Source": "golang.org", "Age": 3}); err != nil {
t.Fatal("failure")
}
if _, err := feeds.Insert(map[string]interface{}{"Title": "Kitkat is here", "Source": "google.com", "Age": 2}); err != nil {
t.Fatal("failure")
}
if _, err := feeds.Insert(map[string]interface{}{"Title": "Good Slackware", "Source": "slackware.com", "Age": 1}); err != nil {
t.Fatal("failure")
}
var query interface{}
if json.Unmarshal([]byte(`[{"eq": "New Go release", "in": ["Title"]}, {"eq": "slackware.com", "in": ["Source"]}]`), &query) != nil {
t.Fatal("failure")
}
queryResult := make(map[int]struct{}) // query result (document IDs) goes into map keys
if err := db.EvalQuery(query, feeds, &queryResult); err != nil {
t.Fatal(err)
}
// Query result are document IDs
for id := range queryResult {
// To get query result document, simply read it
readBack, err := feeds.Read(id)
if err != nil {
panic(err)
}
if !sameMap(readBack, map[string]interface{}{"Title": "New Go release", "Source": "golang.org", "Age": 3}) &&
!sameMap(readBack, map[string]interface{}{"Title": "Good Slackware", "Source": "slackware.com", "Age": 1}) {
t.Fatal(readBack)
}
}
if err := myDB.Close(); err != nil {
t.Fatal(err)
}
}
func embeddedExample() {
// ****************** Collection Management ******************
myDBDir := "/tmp/MyDatabase"
os.RemoveAll(myDBDir)
defer os.RemoveAll(myDBDir)
// (Create if not exist) open a database
myDB, err := db.OpenDB(myDBDir)
if err != nil {
panic(err)
}
// Create two collections: Feeds and Votes
if err := myDB.Create("Feeds"); err != nil {
panic(err)
}
if err := myDB.Create("Votes"); err != nil {
panic(err)
}
// What collections do I now have?
for _, name := range myDB.AllCols() {
fmt.Printf("I have a collection called %s\n", name)
}
// Rename collection "Votes" to "Points"
if err := myDB.Rename("Votes", "Points"); err != nil {
panic(err)
}
// Drop (delete) collection "Points"
if err := myDB.Drop("Points"); err != nil {
panic(err)
}
// Scrub (repair and compact) "Feeds"
if err := myDB.Scrub("Feeds"); err != nil {
panic(err)
}
// ****************** Document Management ******************
// Start using a collection (the reference is valid until DB schema changes or Scrub is carried out)
feeds := myDB.Use("Feeds")
// Insert document (afterwards the docID uniquely identifies the document and will never change)
docID, err := feeds.Insert(map[string]interface{}{
"name": "Go 1.2 is released",
"url": "golang.org"})
if err != nil {
panic(err)
}
// Read document
readBack, err := feeds.Read(docID)
if err != nil {
panic(err)
}
fmt.Println("Document", docID, "is", readBack)
// Update document
err = feeds.Update(docID, map[string]interface{}{
"name": "Go is very popular",
"url": "google.com"})
if err != nil {
panic(err)
}
// Process all documents (note that document order is undetermined)
feeds.ForEachDoc(func(id int, docContent []byte) (willMoveOn bool) {
fmt.Println("Document", id, "is", string(docContent))
return true // move on to the next document OR
return false // do not move on to the next document
})
// Delete document
if err := feeds.Delete(docID); err != nil {
panic(err)
}
// ****************** Index Management ******************
// Indexes assist in many types of queries
// Create index (path leads to document JSON attribute)
if err := feeds.Index([]string{"author", "name", "first_name"}); err != nil {
panic(err)
}
if err := feeds.Index([]string{"Title"}); err != nil {
panic(err)
}
if err := feeds.Index([]string{"Source"}); err != nil {
panic(err)
}
// What indexes do I have on collection A?
for _, path := range feeds.AllIndexes() {
fmt.Printf("I have an index on path %v\n", path)
}
// Remove index
if err := feeds.Unindex([]string{"author", "name", "first_name"}); err != nil {
panic(err)
}
// ****************** Queries ******************
// Prepare some documents for the query
feeds.Insert(map[string]interface{}{"Title": "New Go release", "Source": "golang.org", "Age": 3})
feeds.Insert(map[string]interface{}{"Title": "Kitkat is here", "Source": "google.com", "Age": 2})
feeds.Insert(map[string]interface{}{"Title": "Good Slackware", "Source": "slackware.com", "Age": 1})
var query interface{}
json.Unmarshal([]byte(`[{"eq": "New Go release", "in": ["Title"]}, {"eq": "slackware.com", "in": ["Source"]}]`), &query)
queryResult := make(map[int]struct{}) // query result (document IDs) goes into map keys
if err := db.EvalQuery(query, feeds, &queryResult); err != nil {
panic(err)
}
// Query result are document IDs
for id := range queryResult {
// To get query result document, simply read it
readBack, err := feeds.Read(id)
if err != nil {
panic(err)
}
fmt.Printf("Query returned document %v\n", readBack)
}
// Make sure important transactions are persisted (very expensive call: do NOT invoke too often)
// (A background goroutine is already doing it for you every few seconds)
if err := myDB.Sync(); err != nil {
panic(err)
}
// Gracefully close database, you should call Close on all opened databases
// Otherwise background goroutines will prevent program shutdown
if err := myDB.Close(); err != nil {
panic(err)
}
}
func main() {
var err error
var defaultMaxprocs int
if defaultMaxprocs, err = strconv.Atoi(os.Getenv("GOMAXPROCS")); err != nil {
defaultMaxprocs = runtime.NumCPU()
}
// Parse CLI parameters
var mode, dir string
var port, maxprocs int
var profile bool
flag.StringVar(&mode, "mode", "", "[httpd|bench|bench2|example]")
flag.StringVar(&dir, "dir", "", "(HTTP API) database directory")
flag.IntVar(&port, "port", 8080, "(HTTP API) port number")
flag.StringVar(&webcp.WebCp, "webcp", "admin", "(HTTP API) web control panel route (without leading slash)")
flag.IntVar(&maxprocs, "gomaxprocs", defaultMaxprocs, "GOMAXPROCS")
flag.IntVar(&benchSize, "benchsize", 400000, "Benchmark sample size")
flag.BoolVar(&profile, "profile", false, "Write profiler results to prof.out")
flag.BoolVar(&tdlog.VerboseLog, "verbose", false, "Turn verbose logging on/off")
flag.BoolVar(&benchCleanup, "benchcleanup", true, "Whether to clean up (delete benchmark DB) after benchmark")
flag.Parse()
// User must specify a mode to run
if mode == "" {
flag.PrintDefaults()
return
}
// Set appropriate GOMAXPROCS
runtime.GOMAXPROCS(maxprocs)
tdlog.Noticef("GOMAXPROCS is set to %d", maxprocs)
if maxprocs < runtime.NumCPU() {
tdlog.Noticef("GOMAXPROCS (%d) is less than number of CPUs (%d), this may reduce performance. You can change it via environment variable GOMAXPROCS or by passing CLI parameter -gomaxprocs", maxprocs, runtime.NumCPU())
}
// Start profiler if enabled
if profile {
resultFile, err := os.Create("perf.out")
if err != nil {
log.Panicf("Cannot create profiler result file %s", resultFile)
}
pprof.StartCPUProfile(resultFile)
defer pprof.StopCPUProfile()
}
switch mode {
case "httpd": // Run HTTP API server
if dir == "" {
tdlog.Panicf("Please specify database directory, for example -dir=/tmp/db")
}
if port == 0 {
tdlog.Panicf("Please specify port number, for example -port=8080")
}
db, err := db.OpenDB(dir)
if err != nil {
panic(err)
}
httpapi.Start(db, port)
case "example": // Run embedded usage examples
embeddedExample()
case "bench": // Benchmark scenarios
benchmark()
case "bench2":
benchmark2()
default:
flag.PrintDefaults()
return
}
}
func embeddedExample() {
// ****************** Collection Management ******************
// Create and open database
dir := "/tmp/MyDatabase"
os.RemoveAll(dir)
defer os.RemoveAll(dir)
myDB, err := db.OpenDB(dir)
if err != nil {
panic(err)
}
// Create two collections Feeds and Votes
if err := myDB.Create("Feeds"); err != nil {
panic(err)
}
if err := myDB.Create("Votes"); err != nil {
panic(err)
}
// What collections do I now have?
for name := range myDB.StrCol {
fmt.Printf("I have a collection called %s\n", name)
}
// Rename collection "Votes" to "Points"
if err := myDB.Rename("Votes", "Points"); err != nil {
panic(err)
}
// Drop (delete) collection "Points"
if err := myDB.Drop("Points"); err != nil {
panic(err)
}
// Scrub (repair and compact) "Feeds"
if err := myDB.Scrub("Feeds"); err != nil {
panic(err)
}
// ****************** Document Management ******************
// Start using a collection
feeds := myDB.Use("Feeds")
// Insert document (document must be map[string]interface{})
docID, err := feeds.Insert(map[string]interface{}{
"name": "Go 1.2 is released",
"url": "golang.org"})
if err != nil {
panic(err)
}
// Read document
var readBack interface{}
feeds.Read(docID, &readBack) // pass in document's physical ID
fmt.Println(readBack)
// Update document (document must be map[string]interface{})
newID, err := feeds.Update(docID, map[string]interface{}{
"name": "Go is very popular",
"url": "google.com"})
if err != nil {
panic(err)
}
// Delete document
feeds.Delete(newID)
// Delete document
feeds.Delete(123) // An ID which does not exist does no harm
// ****************** Index Management ******************
// Create index (path leads to document JSON attribute)
if err := feeds.Index([]string{"author", "name", "first_name"}); err != nil {
panic(err)
}
// What indexes do I have on collection A?
for path := range feeds.StrHT {
fmt.Printf("I have an index on path %s\n", path)
}
// Remove index
if err := feeds.Unindex([]string{"author", "name", "first_name"}); err != nil {
panic(err)
}
// ****************** Queries ******************
// Let's prepare a number of docments for a start
feeds.Insert(map[string]interface{}{"Title": "New Go release", "Source": "golang.org", "Age": 3})
feeds.Insert(map[string]interface{}{"Title": "Kitkat is here", "Source": "google.com", "Age": 2})
feeds.Insert(map[string]interface{}{"Title": "Good Slackware", "Source": "slackware.com", "Age": 1})
// There are two ways to construct query - by deserializing tiedot query string
// Or construct a Golang map[string]interface{} (essentially, deserialized JSON)
queryStr := `[{"eq": "New Go release", "in": ["Title"]}, {"eq": "slackware.com", "in": ["Source"]}]`
var query interface{}
json.Unmarshal([]byte(queryStr), &query)
queryResult := make(map[uint64]struct{}) // query result goes into map keys
if err := db.EvalQueryV2(query, feeds, &queryResult); err != nil {
panic(err)
}
// Query results are physical document IDs
for id := range queryResult {
fmt.Printf("Query returned document ID %d\n", id)
}
// To use the document itself, simply read it back
for id := range queryResult {
feeds.Read(id, &readBack)
fmt.Printf("Query returned document %v\n", readBack)
}
// Gracefully close database
myDB.Close()
}
func embeddedExample() {
// ****************** Collection Management ******************
myDBDir := "/tmp/MyDatabase"
os.RemoveAll(myDBDir)
defer os.RemoveAll(myDBDir)
// (Create if not exist) open a database
myDB, err := db.OpenDB(myDBDir)
if err != nil {
panic(err)
}
// Create two collections: Feeds and Votes
if err := myDB.Create("Feeds"); err != nil {
panic(err)
}
if err := myDB.Create("Votes"); err != nil {
panic(err)
}
// What collections do I now have?
for _, name := range myDB.AllCols() {
fmt.Printf("I have a collection called %s\n", name)
}
// Rename collection "Votes" to "Points"
if err := myDB.Rename("Votes", "Points"); err != nil {
panic(err)
}
// Drop (delete) collection "Points"
if err := myDB.Drop("Points"); err != nil {
panic(err)
}
// Scrub (repair and compact) "Feeds"
myDB.Scrub("Feeds")
// ****************** Document Management ******************
// Start using a collection
// (The reference is valid until DB schema changes or Scrub is carried out)
feeds := myDB.Use("Feeds")
// Insert document (document must be map[string]interface{})
docID, err := feeds.Insert(map[string]interface{}{
"name": "Go 1.2 is released",
"url": "golang.org"})
if err != nil {
panic(err)
}
// Read document
readBack, err := feeds.Read(docID)
fmt.Println(readBack)
// Update document (document must be map[string]interface{})
err = feeds.Update(docID, map[string]interface{}{
"name": "Go is very popular",
"url": "google.com"})
if err != nil {
panic(err)
}
// Delete document
feeds.Delete(docID)
// ****************** Index Management ******************
// Secondary indexes assist in many types of queries
// Create index (path leads to document JSON attribute)
if err := feeds.Index([]string{"author", "name", "first_name"}); err != nil {
panic(err)
}
if err := feeds.Index([]string{"Title"}); err != nil {
panic(err)
}
if err := feeds.Index([]string{"Source"}); err != nil {
panic(err)
}
// What indexes do I have on collection A?
for _, path := range feeds.AllIndexes() {
fmt.Printf("I have an index on path %v\n", path)
}
// Remove index
if err := feeds.Unindex([]string{"author", "name", "first_name"}); err != nil {
panic(err)
}
// ****************** Queries ******************
// Prepare some documents for the query
feeds.Insert(map[string]interface{}{"Title": "New Go release", "Source": "golang.org", "Age": 3})
feeds.Insert(map[string]interface{}{"Title": "Kitkat is here", "Source": "google.com", "Age": 2})
feeds.Insert(map[string]interface{}{"Title": "Good Slackware", "Source": "slackware.com", "Age": 1})
feeds.ForEachDoc(true, func(id int, doc []byte) bool {
var data map[string]interface{}
json.Unmarshal(doc, &data)
fmt.Println("data is")
fmt.Println(data)
fmt.Println("data was")
return true
})
var query interface{}
json.Unmarshal([]byte(`[{"eq": "New Go release", "in": ["Title"]}, {"eq": "slackware.com", "in": ["Source"]}]`), &query)
queryResult := make(map[int]struct{}) // query result (document IDs) goes into map keys
if err := db.EvalQuery(query, feeds, &queryResult); err != nil {
panic(err)
}
// Query results are document IDs
for id := range queryResult {
fmt.Printf("Query returned document ID %d\n", id)
}
// To use the document itself, simply read it back
for id := range queryResult {
readBack, err := feeds.Read(id)
if err != nil {
panic(err)
}
fmt.Printf("Query returned document %v\n", readBack)
}
// Make sure important transactions are persisted (very expensive call: do NOT invoke too often)
// (A background goroutine is already doing it for you every few seconds)
if err := myDB.Sync(); err != nil {
panic(err)
}
// Gracefully close database
if err := myDB.Close(); err != nil {
panic(err)
}
}