func Start(dir string, port int, tlsCrt, tlsKey, jwtPubKey, jwtPrivateKey string) { var err error HttpDB, err = db.OpenDB(dir) if err != nil { panic(err) } // These endpoints are always available and do not require JWT auth http.HandleFunc("/", Welcome) http.HandleFunc("/version", Version) http.HandleFunc("/memstats", MemStats) if jwtPrivateKey != "" { // JWT support ServeJWTEnabledEndpoints(jwtPubKey, jwtPrivateKey) } else { // No JWT ServeEndpoints() } if tlsCrt != "" { tdlog.Noticef("Will listen on all interfaces (HTTPS), port %d.", port) if err := http.ListenAndServeTLS(fmt.Sprintf(":%d", port), tlsCrt, tlsKey, nil); err != nil { tdlog.Panicf("Failed to start HTTPS service - %s", err) } } else { tdlog.Noticef("Will listen on all interfaces (HTTP), port %d.", port) http.ListenAndServe(fmt.Sprintf(":%d", port), nil) } }
// Read Linux system VM parameters and print performance configuration advice when necessary. func linuxPerfAdvice() { readFileIntContent := func(filePath string) (contentInt int, err error) { content, err := ioutil.ReadFile(filePath) if err != nil { return } contentInt, err = strconv.Atoi(strings.TrimSpace(string(content))) return } swappiness, err := readFileIntContent("/proc/sys/vm/swappiness") if err != nil { tdlog.Notice("Non-fatal - unable to offer performance advice based on vm.swappiness.") } else if swappiness > 50 { tdlog.Noticef("System vm.swappiness is very high (%d), for optimium performance please lower it to below 50.", swappiness) } dirtyRatio, err := readFileIntContent("/proc/sys/vm/dirty_ratio") if err != nil { tdlog.Notice("Non-fatal - unable to offer performance advice based on vm.dirty_ratio.") } else if dirtyRatio < 50 { tdlog.Noticef("System vm.dirty_ratio is very low (%d), for optimium performance please increase it to above 50.", dirtyRatio) } dirtyBGRatio, err := readFileIntContent("/proc/sys/vm/dirty_background_ratio") if err != nil { tdlog.Notice("Non-fatal - unable to offer performance advice based on vm.dirty_background_ratio.") } else if dirtyBGRatio < 50 { tdlog.Noticef("System vm.dirty_background_ratio is very low (%d), for optimium performance please increase it to above 50.", dirtyBGRatio) } }
// Copy this database into destination directory (for backup). func (db *DB) Dump(dest string) error { db.schemaLock.Lock() defer db.schemaLock.Unlock() cpFun := func(currPath string, info os.FileInfo, err error) error { if err != nil { return err } if info.IsDir() { relPath, err := filepath.Rel(db.path, currPath) if err != nil { return err } destDir := path.Join(dest, relPath) if err := os.MkdirAll(destDir, 0700); err != nil { return err } tdlog.Noticef("Dump: created directory %s", destDir) } else { src, err := os.Open(currPath) if err != nil { return err } relPath, err := filepath.Rel(db.path, currPath) if err != nil { return err } destPath := path.Join(dest, relPath) if _, fileExists := os.Open(destPath); fileExists == nil { return fmt.Errorf("Destination file %s already exists", destPath) } destFile, err := os.Create(destPath) if err != nil { return err } written, err := io.Copy(destFile, src) if err != nil { return err } tdlog.Noticef("Dump: copied file %s, size is %d", destPath, written) } return nil } return filepath.Walk(db.path, cpFun) }
// Scrub a collection - fix corrupted documents and de-fragment free space. func (db *DB) Scrub(name string) error { db.schemaLock.Lock() defer db.schemaLock.Unlock() if _, exists := db.cols[name]; !exists { return fmt.Errorf("Collection %s does not exist", name) } // Prepare a temporary collection in file system tmpColName := fmt.Sprintf("scrub-%s-%d", name, time.Now().UnixNano()) tmpColDir := path.Join(db.path, tmpColName) if err := os.MkdirAll(tmpColDir, 0700); err != nil { return err } // Mirror indexes from original collection for _, idxPath := range db.cols[name].indexPaths { if err := os.MkdirAll(path.Join(tmpColDir, strings.Join(idxPath, INDEX_PATH_SEP)), 0700); err != nil { return err } } // Iterate through all documents and put them into the temporary collection tmpCol, err := OpenCol(db, tmpColName) if err != nil { return err } db.cols[name].forEachDoc(func(id int, doc []byte) bool { var docObj map[string]interface{} if err := json.Unmarshal([]byte(doc), &docObj); err != nil { // Skip corrupted document return true } if err := tmpCol.InsertRecovery(id, docObj); err != nil { tdlog.Noticef("Scrub %s: failed to insert back document %v", name, docObj) } return true }, false) if err := tmpCol.close(); err != nil { return err } // Replace the original collection with the "temporary" one db.cols[name].close() if err := os.RemoveAll(path.Join(db.path, name)); err != nil { return err } if err := os.Rename(path.Join(db.path, tmpColName), path.Join(db.path, name)); err != nil { return err } if db.cols[name], err = OpenCol(db, name); err != nil { return err } return nil }
// Update a document. func (col *Col) Update(id int, doc map[string]interface{}) error { if doc == nil { return fmt.Errorf("Updating %d: input doc may not be nil", id) } docJS, err := json.Marshal(doc) if err != nil { return err } col.db.schemaLock.RLock() part := col.parts[id%col.db.numParts] part.Lock.Lock() // Place lock, read back original document and update if err := part.LockUpdate(id); err != nil { part.Lock.Unlock() col.db.schemaLock.RUnlock() return err } originalB, err := part.Read(id) if err != nil { part.UnlockUpdate(id) part.Lock.Unlock() col.db.schemaLock.RUnlock() return err } var original map[string]interface{} if err = json.Unmarshal(originalB, &original); err != nil { tdlog.Noticef("Will not attempt to unindex document %d during update", id) } if err = part.Update(id, []byte(docJS)); err != nil { part.UnlockUpdate(id) part.Lock.Unlock() col.db.schemaLock.RUnlock() return err } // Done with the collection data, next is to maintain indexed values part.Lock.Unlock() if original != nil { col.unindexDoc(id, original) } col.indexDoc(id, doc) // Done with the document part.Lock.Lock() part.UnlockUpdate(id) part.Lock.Unlock() col.db.schemaLock.RUnlock() return nil }
// Delete a document. func (col *Col) Delete(id int) error { col.db.schemaLock.RLock() part := col.parts[id%col.db.numParts] part.Lock.Lock() // Place lock, read back original document and delete document if err := part.LockUpdate(id); err != nil { part.Lock.Unlock() col.db.schemaLock.RUnlock() return err } originalB, err := part.Read(id) if err != nil { part.UnlockUpdate(id) part.Lock.Unlock() col.db.schemaLock.RUnlock() return err } var original map[string]interface{} if err = json.Unmarshal(originalB, &original); err != nil { tdlog.Noticef("Will not attempt to unindex document %d during delete", id) } if err = part.Delete(id); err != nil { part.UnlockUpdate(id) part.Lock.Unlock() col.db.schemaLock.RUnlock() return err } // Done with the collection data, next is to remove indexed values part.Lock.Unlock() if original != nil { col.unindexDoc(id, original) } part.Lock.Lock() part.UnlockUpdate(id) part.Lock.Unlock() col.db.schemaLock.RUnlock() return nil }
func main() { var err error var defaultMaxprocs int if defaultMaxprocs, err = strconv.Atoi(os.Getenv("GOMAXPROCS")); err != nil { defaultMaxprocs = runtime.NumCPU() } // Parse CLI parameters // General params var mode string var maxprocs int flag.StringVar(&mode, "mode", "", "[httpd|bench|bench2|example]") flag.IntVar(&maxprocs, "gomaxprocs", defaultMaxprocs, "GOMAXPROCS") // Debug params var profile, debug bool flag.BoolVar(&tdlog.VerboseLog, "verbose", false, "Turn verbose logging on/off") flag.BoolVar(&profile, "profile", false, "Write profiler results to prof.out") flag.BoolVar(&debug, "debug", false, "Dump goroutine stack traces upon receiving interrupt signal") // HTTP mode params var dir string var port int var tlsCrt, tlsKey string flag.StringVar(&dir, "dir", "", "(HTTP server) database directory") flag.IntVar(&port, "port", 8080, "(HTTP server) port number") flag.StringVar(&tlsCrt, "tlscrt", "", "(HTTP server) TLS certificate (empty to disable TLS).") flag.StringVar(&tlsKey, "tlskey", "", "(HTTP server) TLS certificate key (empty to disable TLS).") // HTTP + JWT params var jwtPubKey, jwtPrivateKey string flag.StringVar(&jwtPubKey, "jwtpubkey", "", "(HTTP JWT server) Public key for signing tokens (empty to disable JWT)") flag.StringVar(&jwtPrivateKey, "jwtprivatekey", "", "(HTTP JWT server) Private key for decoding tokens (empty to disable JWT)") // Benchmark mode params flag.IntVar(&benchSize, "benchsize", 400000, "Benchmark sample size") flag.BoolVar(&benchCleanup, "benchcleanup", true, "Whether to clean up (delete benchmark DB) after benchmark") flag.Parse() // User must specify a mode to run if mode == "" { flag.PrintDefaults() os.Exit(1) } // Set appropriate GOMAXPROCS runtime.GOMAXPROCS(maxprocs) tdlog.Noticef("GOMAXPROCS is set to %d", maxprocs) // Performance advices if maxprocs < runtime.NumCPU() { tdlog.Noticef("GOMAXPROCS (%d) is less than number of CPUs (%d), this may reduce performance. You can change it via environment variable GOMAXPROCS or by passing CLI parameter -gomaxprocs", maxprocs, runtime.NumCPU()) } linuxPerfAdvice() // Start profiler if enabled if profile { resultFile, err := os.Create("perf.out") if err != nil { tdlog.Noticef("Cannot create profiler result file %s", resultFile) os.Exit(1) } pprof.StartCPUProfile(resultFile) defer pprof.StopCPUProfile() } // Dump goroutine stacktraces upon receiving interrupt signal if debug { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) go func() { for range c { pprof.Lookup("goroutine").WriteTo(os.Stderr, 1) } }() } switch mode { case "httpd": // Run HTTP API server if dir == "" { tdlog.Notice("Please specify database directory, for example -dir=/tmp/db") os.Exit(1) } if port == 0 { tdlog.Notice("Please specify port number, for example -port=8080") os.Exit(1) } if tlsCrt != "" && tlsKey == "" || tlsKey != "" && tlsCrt == "" { tdlog.Notice("To enable HTTPS, please specify both RSA certificate and key file.") os.Exit(1) } if jwtPrivateKey != "" && jwtPubKey == "" || jwtPubKey != "" && jwtPrivateKey == "" { tdlog.Notice("To enable JWT, please specify RSA private and public key.") os.Exit(1) } httpapi.Start(dir, port, tlsCrt, tlsKey, jwtPubKey, jwtPrivateKey) case "example": // Run embedded usage examples embeddedExample() case "bench": // Benchmark scenarios benchmark() case "bench2": benchmark2() default: flag.PrintDefaults() return } }