func Start(dir string, port int, tlsCrt, tlsKey, jwtPubKey, jwtPrivateKey string) {
var err error
HttpDB, err = db.OpenDB(dir)
if err != nil {
panic(err)
}
// These endpoints are always available and do not require JWT auth
http.HandleFunc("/", Welcome)
http.HandleFunc("/version", Version)
http.HandleFunc("/memstats", MemStats)
if jwtPrivateKey != "" {
// JWT support
ServeJWTEnabledEndpoints(jwtPubKey, jwtPrivateKey)
} else {
// No JWT
ServeEndpoints()
}
if tlsCrt != "" {
tdlog.Noticef("Will listen on all interfaces (HTTPS), port %d.", port)
if err := http.ListenAndServeTLS(fmt.Sprintf(":%d", port), tlsCrt, tlsKey, nil); err != nil {
tdlog.Panicf("Failed to start HTTPS service - %s", err)
}
} else {
tdlog.Noticef("Will listen on all interfaces (HTTP), port %d.", port)
http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
}
}
// Read Linux system VM parameters and print performance configuration advice when necessary.
func linuxPerfAdvice() {
readFileIntContent := func(filePath string) (contentInt int, err error) {
content, err := ioutil.ReadFile(filePath)
if err != nil {
return
}
contentInt, err = strconv.Atoi(strings.TrimSpace(string(content)))
return
}
swappiness, err := readFileIntContent("/proc/sys/vm/swappiness")
if err != nil {
tdlog.Notice("Non-fatal - unable to offer performance advice based on vm.swappiness.")
} else if swappiness > 50 {
tdlog.Noticef("System vm.swappiness is very high (%d), for optimium performance please lower it to below 50.", swappiness)
}
dirtyRatio, err := readFileIntContent("/proc/sys/vm/dirty_ratio")
if err != nil {
tdlog.Notice("Non-fatal - unable to offer performance advice based on vm.dirty_ratio.")
} else if dirtyRatio < 50 {
tdlog.Noticef("System vm.dirty_ratio is very low (%d), for optimium performance please increase it to above 50.", dirtyRatio)
}
dirtyBGRatio, err := readFileIntContent("/proc/sys/vm/dirty_background_ratio")
if err != nil {
tdlog.Notice("Non-fatal - unable to offer performance advice based on vm.dirty_background_ratio.")
} else if dirtyBGRatio < 50 {
tdlog.Noticef("System vm.dirty_background_ratio is very low (%d), for optimium performance please increase it to above 50.", dirtyBGRatio)
}
}
// Copy this database into destination directory (for backup).
func (db *DB) Dump(dest string) error {
db.schemaLock.Lock()
defer db.schemaLock.Unlock()
cpFun := func(currPath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
relPath, err := filepath.Rel(db.path, currPath)
if err != nil {
return err
}
destDir := path.Join(dest, relPath)
if err := os.MkdirAll(destDir, 0700); err != nil {
return err
}
tdlog.Noticef("Dump: created directory %s", destDir)
} else {
src, err := os.Open(currPath)
if err != nil {
return err
}
relPath, err := filepath.Rel(db.path, currPath)
if err != nil {
return err
}
destPath := path.Join(dest, relPath)
if _, fileExists := os.Open(destPath); fileExists == nil {
return fmt.Errorf("Destination file %s already exists", destPath)
}
destFile, err := os.Create(destPath)
if err != nil {
return err
}
written, err := io.Copy(destFile, src)
if err != nil {
return err
}
tdlog.Noticef("Dump: copied file %s, size is %d", destPath, written)
}
return nil
}
return filepath.Walk(db.path, cpFun)
}
// Scrub a collection - fix corrupted documents and de-fragment free space.
func (db *DB) Scrub(name string) error {
db.schemaLock.Lock()
defer db.schemaLock.Unlock()
if _, exists := db.cols[name]; !exists {
return fmt.Errorf("Collection %s does not exist", name)
}
// Prepare a temporary collection in file system
tmpColName := fmt.Sprintf("scrub-%s-%d", name, time.Now().UnixNano())
tmpColDir := path.Join(db.path, tmpColName)
if err := os.MkdirAll(tmpColDir, 0700); err != nil {
return err
}
// Mirror indexes from original collection
for _, idxPath := range db.cols[name].indexPaths {
if err := os.MkdirAll(path.Join(tmpColDir, strings.Join(idxPath, INDEX_PATH_SEP)), 0700); err != nil {
return err
}
}
// Iterate through all documents and put them into the temporary collection
tmpCol, err := OpenCol(db, tmpColName)
if err != nil {
return err
}
db.cols[name].forEachDoc(func(id int, doc []byte) bool {
var docObj map[string]interface{}
if err := json.Unmarshal([]byte(doc), &docObj); err != nil {
// Skip corrupted document
return true
}
if err := tmpCol.InsertRecovery(id, docObj); err != nil {
tdlog.Noticef("Scrub %s: failed to insert back document %v", name, docObj)
}
return true
}, false)
if err := tmpCol.close(); err != nil {
return err
}
// Replace the original collection with the "temporary" one
db.cols[name].close()
if err := os.RemoveAll(path.Join(db.path, name)); err != nil {
return err
}
if err := os.Rename(path.Join(db.path, tmpColName), path.Join(db.path, name)); err != nil {
return err
}
if db.cols[name], err = OpenCol(db, name); err != nil {
return err
}
return nil
}
// Update a document.
func (col *Col) Update(id int, doc map[string]interface{}) error {
if doc == nil {
return fmt.Errorf("Updating %d: input doc may not be nil", id)
}
docJS, err := json.Marshal(doc)
if err != nil {
return err
}
col.db.schemaLock.RLock()
part := col.parts[id%col.db.numParts]
part.Lock.Lock()
// Place lock, read back original document and update
if err := part.LockUpdate(id); err != nil {
part.Lock.Unlock()
col.db.schemaLock.RUnlock()
return err
}
originalB, err := part.Read(id)
if err != nil {
part.UnlockUpdate(id)
part.Lock.Unlock()
col.db.schemaLock.RUnlock()
return err
}
var original map[string]interface{}
if err = json.Unmarshal(originalB, &original); err != nil {
tdlog.Noticef("Will not attempt to unindex document %d during update", id)
}
if err = part.Update(id, []byte(docJS)); err != nil {
part.UnlockUpdate(id)
part.Lock.Unlock()
col.db.schemaLock.RUnlock()
return err
}
// Done with the collection data, next is to maintain indexed values
part.Lock.Unlock()
if original != nil {
col.unindexDoc(id, original)
}
col.indexDoc(id, doc)
// Done with the document
part.Lock.Lock()
part.UnlockUpdate(id)
part.Lock.Unlock()
col.db.schemaLock.RUnlock()
return nil
}
// Delete a document.
func (col *Col) Delete(id int) error {
col.db.schemaLock.RLock()
part := col.parts[id%col.db.numParts]
part.Lock.Lock()
// Place lock, read back original document and delete document
if err := part.LockUpdate(id); err != nil {
part.Lock.Unlock()
col.db.schemaLock.RUnlock()
return err
}
originalB, err := part.Read(id)
if err != nil {
part.UnlockUpdate(id)
part.Lock.Unlock()
col.db.schemaLock.RUnlock()
return err
}
var original map[string]interface{}
if err = json.Unmarshal(originalB, &original); err != nil {
tdlog.Noticef("Will not attempt to unindex document %d during delete", id)
}
if err = part.Delete(id); err != nil {
part.UnlockUpdate(id)
part.Lock.Unlock()
col.db.schemaLock.RUnlock()
return err
}
// Done with the collection data, next is to remove indexed values
part.Lock.Unlock()
if original != nil {
col.unindexDoc(id, original)
}
part.Lock.Lock()
part.UnlockUpdate(id)
part.Lock.Unlock()
col.db.schemaLock.RUnlock()
return nil
}
func main() {
var err error
var defaultMaxprocs int
if defaultMaxprocs, err = strconv.Atoi(os.Getenv("GOMAXPROCS")); err != nil {
defaultMaxprocs = runtime.NumCPU()
}
// Parse CLI parameters
// General params
var mode string
var maxprocs int
flag.StringVar(&mode, "mode", "", "[httpd|bench|bench2|example]")
flag.IntVar(&maxprocs, "gomaxprocs", defaultMaxprocs, "GOMAXPROCS")
// Debug params
var profile, debug bool
flag.BoolVar(&tdlog.VerboseLog, "verbose", false, "Turn verbose logging on/off")
flag.BoolVar(&profile, "profile", false, "Write profiler results to prof.out")
flag.BoolVar(&debug, "debug", false, "Dump goroutine stack traces upon receiving interrupt signal")
// HTTP mode params
var dir string
var port int
var tlsCrt, tlsKey string
flag.StringVar(&dir, "dir", "", "(HTTP server) database directory")
flag.IntVar(&port, "port", 8080, "(HTTP server) port number")
flag.StringVar(&tlsCrt, "tlscrt", "", "(HTTP server) TLS certificate (empty to disable TLS).")
flag.StringVar(&tlsKey, "tlskey", "", "(HTTP server) TLS certificate key (empty to disable TLS).")
// HTTP + JWT params
var jwtPubKey, jwtPrivateKey string
flag.StringVar(&jwtPubKey, "jwtpubkey", "", "(HTTP JWT server) Public key for signing tokens (empty to disable JWT)")
flag.StringVar(&jwtPrivateKey, "jwtprivatekey", "", "(HTTP JWT server) Private key for decoding tokens (empty to disable JWT)")
// Benchmark mode params
flag.IntVar(&benchSize, "benchsize", 400000, "Benchmark sample size")
flag.BoolVar(&benchCleanup, "benchcleanup", true, "Whether to clean up (delete benchmark DB) after benchmark")
flag.Parse()
// User must specify a mode to run
if mode == "" {
flag.PrintDefaults()
os.Exit(1)
}
// Set appropriate GOMAXPROCS
runtime.GOMAXPROCS(maxprocs)
tdlog.Noticef("GOMAXPROCS is set to %d", maxprocs)
// Performance advices
if maxprocs < runtime.NumCPU() {
tdlog.Noticef("GOMAXPROCS (%d) is less than number of CPUs (%d), this may reduce performance. You can change it via environment variable GOMAXPROCS or by passing CLI parameter -gomaxprocs", maxprocs, runtime.NumCPU())
}
linuxPerfAdvice()
// Start profiler if enabled
if profile {
resultFile, err := os.Create("perf.out")
if err != nil {
tdlog.Noticef("Cannot create profiler result file %s", resultFile)
os.Exit(1)
}
pprof.StartCPUProfile(resultFile)
defer pprof.StopCPUProfile()
}
// Dump goroutine stacktraces upon receiving interrupt signal
if debug {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func() {
for range c {
pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
}
}()
}
switch mode {
case "httpd":
// Run HTTP API server
if dir == "" {
tdlog.Notice("Please specify database directory, for example -dir=/tmp/db")
os.Exit(1)
}
if port == 0 {
tdlog.Notice("Please specify port number, for example -port=8080")
os.Exit(1)
}
if tlsCrt != "" && tlsKey == "" || tlsKey != "" && tlsCrt == "" {
tdlog.Notice("To enable HTTPS, please specify both RSA certificate and key file.")
os.Exit(1)
}
if jwtPrivateKey != "" && jwtPubKey == "" || jwtPubKey != "" && jwtPrivateKey == "" {
tdlog.Notice("To enable JWT, please specify RSA private and public key.")
os.Exit(1)
}
httpapi.Start(dir, port, tlsCrt, tlsKey, jwtPubKey, jwtPrivateKey)
case "example":
// Run embedded usage examples
embeddedExample()
case "bench":
// Benchmark scenarios
benchmark()
case "bench2":
benchmark2()
default:
flag.PrintDefaults()
return
}
}