// Ensure that a re-opened database is consistent. func TestOpen_Check(t *testing.T) { path := tempfile() db, err := bolt.Open(path, 0666, nil) if err != nil { t.Fatal(err) } if err := db.View(func(tx *bolt.Tx) error { return <-tx.Check() }); err != nil { t.Fatal(err) } if err := db.Close(); err != nil { t.Fatal(err) } db, err = bolt.Open(path, 0666, nil) if err != nil { t.Fatal(err) } if err := db.View(func(tx *bolt.Tx) error { return <-tx.Check() }); err != nil { t.Fatal(err) } if err := db.Close(); err != nil { t.Fatal(err) } }
// Ensure that opening an already open database file will wait until its closed. func TestOpen_Wait(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("timeout not supported on windows") } if runtime.GOOS == "solaris" { t.Skip("solaris fcntl locks don't support intra-process locking") } path := tempfile() defer os.Remove(path) // Open a data file. db0, err := bolt.Open(path, 0666, nil) assert(t, db0 != nil, "") ok(t, err) // Close it in just a bit. time.AfterFunc(100*time.Millisecond, func() { db0.Close() }) // Attempt to open the database again. start := time.Now() db1, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 200 * time.Millisecond}) assert(t, db1 != nil, "") ok(t, err) assert(t, time.Since(start) > 100*time.Millisecond, "") }
func ExampleTx_CopyFile() { // Open the database. db, _ := bolt.Open(tempfile(), 0666, nil) defer os.Remove(db.Path()) defer db.Close() // Create a bucket and a key. db.Update(func(tx *bolt.Tx) error { tx.CreateBucket([]byte("widgets")) tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte("bar")) return nil }) // Copy the database to another file. toFile := tempfile() db.View(func(tx *bolt.Tx) error { return tx.CopyFile(toFile, 0666) }) defer os.Remove(toFile) // Open the cloned database. db2, _ := bolt.Open(toFile, 0666, nil) defer db2.Close() // Ensure that the key exists in the copy. db2.View(func(tx *bolt.Tx) error { value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) fmt.Printf("The value for 'foo' in the clone is: %s\n", value) return nil }) // Output: // The value for 'foo' in the clone is: bar }
// Ensure that opening an already open database file will wait until its closed. func TestOpen_Wait(t *testing.T) { if runtime.GOOS == "solaris" { t.Skip("solaris fcntl locks don't support intra-process locking") } path := tempfile() // Open a data file. db0, err := bolt.Open(path, 0666, nil) if err != nil { t.Fatal(err) } // Close it in just a bit. time.AfterFunc(100*time.Millisecond, func() { _ = db0.Close() }) // Attempt to open the database again. start := time.Now() db1, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 200 * time.Millisecond}) if err != nil { t.Fatal(err) } else if time.Since(start) <= 100*time.Millisecond { t.Fatal("expected to wait at least timeout duration") } if err := db1.Close(); err != nil { t.Fatal(err) } }
// Run executes the command. func (cmd *CompactCommand) Run(args ...string) (err error) { // Parse flags. fs := flag.NewFlagSet("", flag.ContinueOnError) fs.SetOutput(ioutil.Discard) fs.StringVar(&cmd.DstPath, "o", "", "") fs.Int64Var(&cmd.TxMaxSize, "tx-max-size", 65536, "") if err := fs.Parse(args); err == flag.ErrHelp { fmt.Fprintln(cmd.Stderr, cmd.Usage()) return ErrUsage } else if err != nil { return err } else if cmd.DstPath == "" { return fmt.Errorf("output file required") } // Require database paths. cmd.SrcPath = fs.Arg(0) if cmd.SrcPath == "" { return ErrPathRequired } // Ensure source file exists. fi, err := os.Stat(cmd.SrcPath) if os.IsNotExist(err) { return ErrFileNotFound } else if err != nil { return err } initialSize := fi.Size() // Open source database. src, err := bolt.Open(cmd.SrcPath, 0444, nil) if err != nil { return err } defer src.Close() // Open destination database. dst, err := bolt.Open(cmd.DstPath, fi.Mode(), nil) if err != nil { return err } defer dst.Close() // Run compaction. if err := cmd.compact(dst, src); err != nil { return err } // Report stats on new size. fi, err = os.Stat(cmd.DstPath) if err != nil { return err } else if fi.Size() == 0 { return fmt.Errorf("zero db size") } fmt.Fprintf(cmd.Stdout, "%d -> %d bytes (gain=%.2fx)\n", initialSize, fi.Size(), float64(initialSize)/float64(fi.Size())) return nil }
func migrateDatabase(c *gin.Context) { fromDB := strings.ToLower(c.DefaultQuery("from", "noneasdf")) toDB := strings.ToLower(c.DefaultQuery("to", "noneasdf")) Debug.Printf("Migrating %s to %s.\n", fromDB, toDB) if !exists(path.Join(RuntimeArgs.SourcePath, fromDB+".db")) { c.JSON(http.StatusOK, gin.H{"success": false, "message": "Can't migrate from " + fromDB + ", it does not exist."}) return } if !exists(path.Join(RuntimeArgs.SourcePath, toDB)) { CopyFile(path.Join(RuntimeArgs.SourcePath, fromDB+".db"), path.Join(RuntimeArgs.SourcePath, toDB+".db")) } else { db, err := bolt.Open(path.Join(RuntimeArgs.SourcePath, fromDB+".db"), 0664, nil) if err != nil { log.Fatal(err) } defer db.Close() db2, err := bolt.Open(path.Join(RuntimeArgs.SourcePath, toDB+".db"), 0664, nil) if err != nil { log.Fatal(err) } defer db2.Close() db2.Update(func(tx *bolt.Tx) error { bucket, err := tx.CreateBucketIfNotExists([]byte("fingerprints")) if err != nil { return fmt.Errorf("create bucket: %s", err) } db.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte("fingerprints")) c := b.Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { bucket.Put(k, v) } return nil }) return nil }) db2.Update(func(tx *bolt.Tx) error { bucket, err := tx.CreateBucketIfNotExists([]byte("fingerprints-track")) if err != nil { return fmt.Errorf("create bucket: %s", err) } db.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte("fingerprints-track")) c := b.Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { bucket.Put(k, v) } return nil }) return nil }) } c.JSON(http.StatusOK, gin.H{"success": true, "message": "Successfully migrated " + fromDB + " to " + toDB}) }
func ExampleTx_CopyFile() { // Open the database. db, err := bolt.Open(tempfile(), 0666, nil) if err != nil { log.Fatal(err) } defer os.Remove(db.Path()) // Create a bucket and a key. if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { return err } if err := b.Put([]byte("foo"), []byte("bar")); err != nil { return err } return nil }); err != nil { log.Fatal(err) } // Copy the database to another file. toFile := tempfile() if err := db.View(func(tx *bolt.Tx) error { return tx.CopyFile(toFile, 0666) }); err != nil { log.Fatal(err) } defer os.Remove(toFile) // Open the cloned database. db2, err := bolt.Open(toFile, 0666, nil) if err != nil { log.Fatal(err) } // Ensure that the key exists in the copy. if err := db2.View(func(tx *bolt.Tx) error { value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) fmt.Printf("The value for 'foo' in the clone is: %s\n", value) return nil }); err != nil { log.Fatal(err) } // Close database to release file lock. if err := db.Close(); err != nil { log.Fatal(err) } if err := db2.Close(); err != nil { log.Fatal(err) } // Output: // The value for 'foo' in the clone is: bar }
func (b *backend) Defrag() error { // TODO: make this non-blocking? // lock batchTx to ensure nobody is using previous tx, and then // close previous ongoing tx. b.batchTx.Lock() defer b.batchTx.Unlock() // lock database after lock tx to avoid deadlock. b.mu.Lock() defer b.mu.Unlock() b.batchTx.commit(true) b.batchTx.tx = nil tmpdb, err := bolt.Open(b.db.Path()+".tmp", 0600, boltOpenOptions) if err != nil { return err } err = defragdb(b.db, tmpdb, defragLimit) if err != nil { tmpdb.Close() os.RemoveAll(tmpdb.Path()) return err } dbp := b.db.Path() tdbp := tmpdb.Path() err = b.db.Close() if err != nil { log.Fatalf("backend: cannot close database (%s)", err) } err = tmpdb.Close() if err != nil { log.Fatalf("backend: cannot close database (%s)", err) } err = os.Rename(tdbp, dbp) if err != nil { log.Fatalf("backend: cannot rename database (%s)", err) } b.db, err = bolt.Open(dbp, 0600, boltOpenOptions) if err != nil { log.Panicf("backend: cannot open database at %s (%v)", dbp, err) } b.batchTx.tx, err = b.db.Begin(true) if err != nil { log.Fatalf("backend: cannot begin tx (%s)", err) } return nil }
// Ensure that a database that is too small returns an error. func TestOpen_FileTooSmall(t *testing.T) { path := tempfile() db, err := bolt.Open(path, 0666, nil) ok(t, err) db.Close() // corrupt the database ok(t, os.Truncate(path, int64(os.Getpagesize()))) db, err = bolt.Open(path, 0666, nil) equals(t, errors.New("file size too small"), err) }
// Ensure that a re-opened database is consistent. func TestOpen_Check(t *testing.T) { path := tempfile() db, err := bolt.Open(path, 0666, nil) ok(t, err) ok(t, db.View(func(tx *bolt.Tx) error { return <-tx.Check() })) db.Close() db, err = bolt.Open(path, 0666, nil) ok(t, err) ok(t, db.View(func(tx *bolt.Tx) error { return <-tx.Check() })) db.Close() }
func GetBoltPageSaver(indexFilename, redirFilename string) (PageSaver, error) { index, err := bolt.Open(indexFilename, 0600, nil) if err != nil { return nil, err } redir, err := bolt.Open(redirFilename, 0600, nil) if err != nil { return nil, err } pageLoader := boltLoader{index, redir, sync.WaitGroup{}, false, sync.Mutex{}} return &pageLoader, nil }
func GetBoltPageLoader() (PageLoader, error) { index, err := bolt.Open(DefaultIndexName, 0600, &bolt.Options{ReadOnly: true}) if err != nil { return nil, err } redir, err := bolt.Open(DefaultRedirName, 0600, &bolt.Options{ReadOnly: true}) if err != nil { return nil, err } pageLoader := boltLoader{index, redir, sync.WaitGroup{}, false, sync.Mutex{}} return &pageLoader, nil }
// CreatePlaylist function creates a playlist item in the database and // also creates it in the filesystem. // It receives the playlist name and returns the modified name and an // error if something went wrong. func CreatePlaylist(name, mPoint string) (string, error) { glog.Infof("Creating Playlist with name: %s\n", name) name = GetCompatibleString(name) db, err := bolt.Open(config.DbPath, 0600, nil) if err != nil { return "", err } defer db.Close() err = db.Update(func(tx *bolt.Tx) error { root, err := tx.CreateBucketIfNotExists([]byte("Playlists")) if err != nil { glog.Errorf("Error creating Playlists bucket: %s\n", err) return err } _, err = root.CreateBucketIfNotExists([]byte(name)) if err != nil { glog.Errorf("Error creating %s bucket: %s\n", name, err) return err } return nil }) if err != nil { return "", err } return name, err }
func read(dbPath string, mapPop bool) { opt := &bolt.Options{Timeout: 5 * time.Minute, ReadOnly: true} if mapPop { fmt.Println("read with MAP_POPULATE flag...") opt = &bolt.Options{Timeout: 5 * time.Minute, ReadOnly: true, MmapFlags: syscall.MAP_POPULATE} } else { fmt.Println("read without MAP_POPULATE flag...") } to := time.Now() db, err := bolt.Open(dbPath, 0600, opt) if err != nil { panic(err) } defer db.Close() fmt.Println("bolt.Open took", time.Since(to)) tr := time.Now() tx, err := db.Begin(writable) if err != nil { panic(err) } defer tx.Rollback() bk := tx.Bucket([]byte(bucketName)) c := bk.Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { // fmt.Printf("%s ---> %s.\n", k, v) _ = k _ = v } fmt.Println("bolt read took:", time.Since(tr)) }
func main() { items := []TestMessage_MsgItem{ TestMessage_MsgItem{ Id: 1, ItemName: "aaaaa", ItemValue: 333, ItemType: 111, }, } msg := TestMessage{ ClientName: "upccup", Description: "test proto", ClientId: 1, Messageitems: items, } db, err := bolt.Open("bolt.db", 0644, nil) if err != nil { log.Fatal(err) } if err := PutMessages(db, &msg); err != nil { log.Fatal(err) } queryMsg := GetMessage(db) log.Printf("query msg: %+v", queryMsg) log.Printf("put msg: %+v", msg) }
// NewTestDB returns a new instance of TestDB. func NewTestDB() *TestDB { db, err := bolt.Open(tempfile(), 0666, nil) if err != nil { panic("cannot open db: " + err.Error()) } return &TestDB{db} }
// testBoltStorage creates a temporary bolt database. In order to clean // resources, stop function must be run at the end of each test. func testBoltStorage() (st storage.ValueInterface, stop func(), err error) { testpath, err := ioutil.TempDir("", "machinegroup") if err != nil { return nil, nil, err } db, err := bolt.Open(filepath.Join(testpath, "test.db"), 0600, nil) if err != nil { return nil, nil, err } stop = func() { db.Close() os.RemoveAll(testpath) } bstorage, err := storage.NewBoltStorageBucket(db, []byte("klient")) if err != nil { stop() return nil, nil, err } return &storage.EncodingStorage{ Interface: bstorage, }, stop, nil }
// ListPlaylists function returns all the names of the playlists available // in the MuLi system. // It receives no arguments and returns a slice of Dir objects to list // all the available playlists and the error if there is any. func ListPlaylists() ([]fuse.Dirent, error) { glog.Info("Entered list playlists.") db, err := bolt.Open(config.DbPath, 0600, nil) if err != nil { return nil, err } defer db.Close() var a []fuse.Dirent err = db.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte("Playlists")) if b == nil { glog.Infof("There is no Playlists bucket.") return nil } c := b.Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { if v == nil { var node fuse.Dirent node.Name = string(k) node.Type = fuse.DT_Dir a = append(a, node) } } return nil }) return a, nil }
func main() { flag.Parse() if *NAME == "" { log.Fatal("name required") } if *MEMORY == 0 { log.Fatal("memory required") } if *CPUS == 0 { log.Fatal("cpus required") } db, err := bolt.Open(*DB_PATH, 0600, nil) if err != nil { log.WithError(err).Fatal("failed to open database") } planrep := dal.NewBoltPlanrep(db) plan := &models.Plan{ Name: *NAME, Memory: *MEMORY * 1024 * 1024, Cpus: *CPUS, DiskSize: *DISK * 1024 * 1024 * 1024, } if err := planrep.Add(plan); err != nil { log.WithError(err).WithField("plan", plan).Fatal("failed to add ip address") } }
func main() { err := readConfig() if err != nil { os.Exit(1) } // Create directory where files will be saved if Config.Path == "" { fmt.Println("'Path' is not defined in configuration. Fallback to 'files'") Config.Path = "files" } os.Mkdir(Config.Path, 0750) db, err := bolt.Open("files.db", 0644, nil) if err != nil { log.Fatal(err) } defer db.Close() InitSessionStore() http.HandleFunc("/", indexHandler) http.HandleFunc("/login", loginHandler) http.HandleFunc("/logout", logoutHandler) http.HandleFunc("/userspace", userspaceHandler) http.HandleFunc("/upload", uploadHandler) http.HandleFunc("/static/", staticHandler) err = http.ListenAndServe(":9090", nil) if err != nil { log.Fatal("ListenAndServe: ", err) } }
func main() { fmt.Println("* Booting cities service...") fmt.Println("* Loading configuration...") options := config.Load("config.json") fmt.Println("* Connecting to the database...") db, err := bolt.Open("cities.db", 0600, nil) if err != nil { panic(fmt.Sprintf("[DB] Couldn't connect to the db: %v", err)) } c := cache.New() parsingDone := make(chan bool, 1) if ds.GetAppStatus(db).IsIndexed() { fmt.Println("[PARSER] Skipping, already done") parsingDone <- true } else { go parser.Scan( db, parsingDone, options.Locales, options.MinPopulation, options.CountriesFile, options.CitiesFile, options.AlternateNamesFile, ) } <-parsingDone fmt.Println("[CACHE] Warming up...") warmUpSearchCache(db, c, options.Locales, 5) fmt.Println("[CACHE] Warming up done") fmt.Printf("* Listening on port %s\n\n", options.Port) log.Fatal(Server(db, options, c).ListenAndServe()) }
func ExampleDB_View() { // Open the database. db, _ := bolt.Open(tempfile(), 0666, nil) defer os.Remove(db.Path()) defer db.Close() // Insert data into a bucket. db.Update(func(tx *bolt.Tx) error { tx.CreateBucket([]byte("people")) b := tx.Bucket([]byte("people")) b.Put([]byte("john"), []byte("doe")) b.Put([]byte("susy"), []byte("que")) return nil }) // Access data from within a read-only transactional block. db.View(func(tx *bolt.Tx) error { v := tx.Bucket([]byte("people")).Get([]byte("john")) fmt.Printf("John's last name is %s.\n", v) return nil }) // Output: // John's last name is doe. }
func ExampleDB_Update() { // Open the database. db, _ := bolt.Open(tempfile(), 0666, nil) defer os.Remove(db.Path()) defer db.Close() // Execute several commands within a write transaction. err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { return err } if err := b.Put([]byte("foo"), []byte("bar")); err != nil { return err } return nil }) // If our transactional block didn't return an error then our data is saved. if err == nil { db.View(func(tx *bolt.Tx) error { value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) fmt.Printf("The value of 'foo' is: %s\n", value) return nil }) } // Output: // The value of 'foo' is: bar }
func Benchmark100CountersWith1000IncrementsEach(t *testing.B) { r := rand.New(rand.NewSource(438)) for i := 0; i < 100; i++ { bucket := "count" + strconv.Itoa(i) for i := 0; i < 1000; i++ { a := int64(r.Uint32() % 1000) counters[bucket] = a } } var ( buff bytes.Buffer err error ) Config.StoreDb = "/tmp/stats_test.db" removeFile(Config.StoreDb) dbHandle, err = bolt.Open(Config.StoreDb, 0644, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { log.Fatalf("Error opening %s (%s)\n", Config.StoreDb, err) } // dbHandle.NoSync = true defer closeAndRemove(dbHandle, Config.StoreDb) t.ResetTimer() for i := 0; i < t.N; i++ { processCounters(&buff, time.Now().Unix(), true, "external", dbHandle) buff = bytes.Buffer{} } }
func ExampleDB_Begin_ReadOnly() { // Open the database. db, _ := bolt.Open(tempfile(), 0666, nil) defer os.Remove(db.Path()) defer db.Close() // Create a bucket. db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucket([]byte("widgets")) return err }) // Create several keys in a transaction. tx, _ := db.Begin(true) b := tx.Bucket([]byte("widgets")) b.Put([]byte("john"), []byte("blue")) b.Put([]byte("abby"), []byte("red")) b.Put([]byte("zephyr"), []byte("purple")) tx.Commit() // Iterate over the values in sorted key order. tx, _ = db.Begin(false) c := tx.Bucket([]byte("widgets")).Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { fmt.Printf("%s likes %s\n", k, v) } tx.Rollback() // Output: // abby likes red // john likes blue // zephyr likes purple }
// Open opens or creates a local storage database with given path. func (driver Driver) Open(dbPath string) (engine.DB, error) { base := path.Dir(dbPath) os.MkdirAll(base, 0755) d, err := bolt.Open(dbPath, 0600, nil) if err != nil { return nil, err } tx, err := d.Begin(true) if err != nil { return nil, err } if _, err = tx.CreateBucketIfNotExists(bucketName); err != nil { tx.Rollback() return nil, err } if err = tx.Commit(); err != nil { return nil, err } return &db{d}, nil }
/* getInfoDB returns a singleton of the infoDB */ func getInfoDB() (*bolt.DB, error) { if db != nil { return db, nil } var err error infoDir := conf.InfoDir err = os.MkdirAll(infoDir, 0777) if err != nil { return nil, err } infoDBPath := filepath.Join(infoDir, "info.db") dbOptions := &bolt.Options{Timeout: 1 * time.Second} db, err = bolt.Open(infoDBPath, 0600, dbOptions) if err != nil { return nil, err } err = db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucketIfNotExists([]byte("info")) if err != nil { return err } return nil }) if err != nil { return nil, err } return db, nil }
// ReadReport gets the ids that needs to be imported from the report already in disk and return the records in a way that can be easily read it func ReadReport(dbname string) (map[string][]string, error) { //(map[string]map[string]interface{}, error) { maa := make(map[string][]string) //make(map[string]map[string]interface{}) db, err := bolt.Open(dbname, 0600, nil) if err != nil { log.Error(uuid, "report.record", err, "Opening bolt database.") } defer db.Close() err = db.View(func(tx *bolt.Tx) error { var err error err = tx.ForEach(func(bucketname []byte, b *bolt.Bucket) error { m, err := GetRecordsForBucket(b) if err != nil { return err } s := make([]string, 0, len(m)) for i := range m { s = append(s, i) } maa[string(bucketname)] = s return err }) return err }) return maa, err }
func getFromCache(repo string) (checksResp, error) { // try and fetch from boltdb db, err := bolt.Open(DBPath, 0600, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { return checksResp{}, fmt.Errorf("failed to open bolt database during GET: %v", err) } defer db.Close() resp := checksResp{} err = db.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(RepoBucket)) if b == nil { return errors.New("No repo bucket") } cached := b.Get([]byte(repo)) if cached == nil { return fmt.Errorf("%q not found in cache", repo) } err = json.Unmarshal(cached, &resp) if err != nil { return fmt.Errorf("failed to parse JSON for %q in cache", repo) } return nil }) if err != nil { return resp, err } resp.LastRefresh = resp.LastRefresh.UTC() return resp, nil }
func (s *boltStore) Open() (err error) { if s.opened { return } // Create the store directory if doesnt exist storeDir := filepath.Dir(s.storePath) if _, err = os.Stat(storeDir); os.IsNotExist(err) { err = os.MkdirAll(storeDir, 0755) if err != nil { return } } s.db, err = bolt.Open(s.storePath, 0660, nil) if err != nil { return } // Initialize all required buckets err = s.db.Update(func(tx *bolt.Tx) (err error) { s.bucket, err = tx.CreateBucketIfNotExists(s.bucketName) return err }) if err == nil { s.opened = true } return }