// createDB creates a new db instance and returns a teardown function the caller // should invoke when done testing to clean up. The close flag indicates // whether or not the teardown function should sync and close the database // during teardown. func createDB(dbType, dbName string, close bool) (btcdb.Db, func(), error) { // Handle memory database specially since it doesn't need the disk // specific handling. if dbType == "memdb" { db, err := btcdb.CreateDB(dbType) if err != nil { return nil, nil, fmt.Errorf("error creating db: %v", err) } // Setup a teardown function for cleaning up. This function is // returned to the caller to be invoked when it is done testing. teardown := func() { if close { db.Close() } } return db, teardown, nil } // Create the root directory for test databases. if !fileExists(testDbRoot) { if err := os.MkdirAll(testDbRoot, 0700); err != nil { err := fmt.Errorf("unable to create test db "+ "root: %v", err) return nil, nil, err } } // Create a new database to store the accepted blocks into. dbPath := filepath.Join(testDbRoot, dbName) _ = os.RemoveAll(dbPath) db, err := btcdb.CreateDB(dbType, dbPath) if err != nil { return nil, nil, fmt.Errorf("error creating db: %v", err) } // Setup a teardown function for cleaning up. This function is // returned to the caller to be invoked when it is done testing. teardown := func() { dbVersionPath := filepath.Join(testDbRoot, dbName+".ver") if close { db.Sync() db.Close() } os.RemoveAll(dbPath) os.Remove(dbVersionPath) os.RemoveAll(testDbRoot) } return db, teardown, nil }
// This example demonstrates creating a new database and inserting the genesis // block into it. func ExampleCreateDB() { // Notice in these example imports that the memdb driver is loaded. // Ordinarily this would be whatever driver(s) your application // requires. // import ( // "github.com/conformal/btcdb" // _ "github.com/conformal/btcdb/memdb" // ) // Create a database and schedule it to be closed on exit. This example // uses a memory-only database to avoid needing to write anything to // the disk. Typically, you would specify a persistent database driver // such as "leveldb" and give it a database name as the second // parameter. db, err := btcdb.CreateDB("memdb") if err != nil { fmt.Println(err) return } defer db.Close() // Insert the main network genesis block. genesis := btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock) newHeight, err := db.InsertBlock(genesis) if err != nil { fmt.Println(err) return } fmt.Println("New height:", newHeight) // Output: // New height: 0 }
// setupBlockDB loads (or creates when needed) the block database taking into // account the selected database backend. It also contains additional logic // such warning the user if there are multiple databases which consume space on // the file system and ensuring the regression test database is clean when in // regression test mode. func setupBlockDB() (btcdb.Db, error) { // The memdb backend does not have a file path associated with it, so // handle it uniquely. We also don't want to worry about the multiple // database type warnings when running with the memory database. if cfg.DbType == "memdb" { btcdLog.Infof("Creating block database in memory.") db, err := btcdb.CreateDB(cfg.DbType) if err != nil { return nil, err } return db, nil } warnMultipeDBs() // The database name is based on the database type. dbPath := blockDbPath(cfg.DbType) // The regression test is special in that it needs a clean database for // each run, so remove it now if it already exists. removeRegressionDB(dbPath) btcdLog.Infof("Loading block database from '%s'", dbPath) db, err := btcdb.OpenDB(cfg.DbType, dbPath) if err != nil { // Return the error if it's not because the database // doesn't exist. if err != btcdb.DbDoesNotExist { return nil, err } // Create the db if it does not exist. err = os.MkdirAll(cfg.DataDir, 0700) if err != nil { return nil, err } db, err = btcdb.CreateDB(cfg.DbType, dbPath) if err != nil { return nil, err } } return db, nil }
// loadBlockDB opens the block database and returns a handle to it. func loadBlockDB() (btcdb.Db, error) { // The database name is based on the database type. dbName := blockDbNamePrefix + "_" + cfg.DbType if cfg.DbType == "sqlite" { dbName = dbName + ".db" } dbPath := filepath.Join(cfg.DataDir, dbName) // The regression test is special in that it needs a clean database for // each run, so remove it now if it already exists. removeRegressionDB(dbPath) log.Infof("[BMGR] Loading block database from '%s'", dbPath) db, err := btcdb.OpenDB(cfg.DbType, dbPath) if err != nil { // Return the error if it's not because the database doesn't // exist. if err != btcdb.DbDoesNotExist { return nil, err } // Create the db if it does not exist. err = os.MkdirAll(cfg.DataDir, 0700) if err != nil { return nil, err } db, err = btcdb.CreateDB(cfg.DbType, dbPath) if err != nil { return nil, err } } // Get the latest block height from the database. _, height, err := db.NewestSha() if err != nil { db.Close() return nil, err } // Insert the appropriate genesis block for the bitcoin network being // connected to if needed. if height == -1 { genesis := btcutil.NewBlock(activeNetParams.genesisBlock) _, err := db.InsertBlock(genesis) if err != nil { db.Close() return nil, err } log.Infof("[BMGR] Inserted genesis block %v", activeNetParams.genesisHash) height = 0 } log.Infof("[BMGR] Block database loaded with block height %d", height) return db, nil }
// exampleLoadDB is used in the example to elide the setup code. func exampleLoadDB() (btcdb.Db, error) { db, err := btcdb.CreateDB("memdb") if err != nil { return nil, err } // Insert the main network genesis block. genesis := btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock) _, err = db.InsertBlock(genesis) if err != nil { return nil, err } return db, err }
func TestBdb(t *testing.T) { // Ignore db remove errors since it means we didn't have an old one. _ = os.Remove("tstdb1") db, err := btcdb.CreateDB("sqlite", "tstdb1") if err != nil { t.Errorf("Failed to open test database %v", err) return } defer os.Remove("tstdb1") for i := range testShas { var previous btcwire.ShaHash if i == 0 { previous = btcwire.GenesisHash } else { previous = testShas[i-1] } _, err := db.InsertBlockData(&testShas[i], &previous, 1, zeroBlock) if err != nil { t.Errorf("Failed to insert testSha %d. Error: %v", i, err) return } testFetch(t, db, testShas[0:i+1], "pre sync ") } // XXX insert enough so that we hit the transaction limit // XXX try and insert a with a bad previous db.Sync() testFetch(t, db, testShas, "post sync") for i := len(testShas) - 1; i >= 0; i-- { err := db.DropAfterBlockBySha(testShas[i]) if err != nil { t.Errorf("drop after %d failed %v", i, err) break } testFetch(t, db, testShas[:i+1], fmt.Sprintf("post DropAfter for sha %d", i)) } // Just tests that it doesn't crash, no return value db.Close() }
func TestEmptyDB(t *testing.T) { dbname := "tstdbempty" dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) _ = os.RemoveAll(dbnamever) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer os.RemoveAll(dbname) defer os.RemoveAll(dbnamever) sha, height, err := db.NewestSha() if !sha.IsEqual(&btcwire.ShaHash{}) { t.Errorf("sha not zero hash") } if height != -1 { t.Errorf("height not -1 %v", height) } // This is a reopen test if err := db.Close(); err != nil { t.Errorf("Close: unexpected error: %v", err) } db, err = btcdb.OpenDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer func() { if err := db.Close(); err != nil { t.Errorf("Close: unexpected error: %v", err) } }() sha, height, err = db.NewestSha() if !sha.IsEqual(&btcwire.ShaHash{}) { t.Errorf("sha not zero hash") } if height != -1 { t.Errorf("height not -1 %v", height) } }
// TestCreateOpenUnsupported ensures that attempting to create or open an // unsupported database type is handled properly. func TestCreateOpenUnsupported(t *testing.T) { // Ensure creating a database with an unsupported type fails with the // expected error. dbType := "unsupported" _, err := btcdb.CreateDB(dbType, "unsupportedcreatetest") if err != btcdb.ErrDbUnknownType { t.Errorf("TestCreateOpenUnsupported: expected error not "+ "received - got: %v, want %v", err, btcdb.ErrDbUnknownType) return } // Ensure opening a database with the new type fails with the expected // error. _, err = btcdb.OpenDB(dbType, "unsupportedopentest") if err != btcdb.ErrDbUnknownType { t.Errorf("TestCreateOpenUnsupported: expected error not "+ "received - got: %v, want %v", err, btcdb.ErrDbUnknownType) return } }
// This example demonstrates how to create a new chain instance and use // ProcessBlock to attempt to attempt add a block to the chain. As the package // overview documentation describes, this includes all of the Bitcoin consensus // rules. This example intentionally attempts to insert a duplicate genesis // block to illustrate how an invalid block is handled. func ExampleBlockChain_ProcessBlock() { // Create a new database to store the accepted blocks into. Typically // this would be opening an existing database and would not use memdb // which is a memory-only database backend, but we create a new db // here so this is a complete working example. db, err := btcdb.CreateDB("memdb") if err != nil { fmt.Printf("Failed to create database: %v\n", err) return } defer db.Close() // Insert the main network genesis block. This is part of the initial // database setup. Like above, this typically would not be needed when // opening an existing database. genesisBlock := btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock) _, err = db.InsertBlock(genesisBlock) if err != nil { fmt.Printf("Failed to insert genesis block: %v\n", err) return } // Create a new BlockChain instance using the underlying database for // the main bitcoin network and ignore notifications. chain := btcchain.New(db, &btcnet.MainNetParams, nil) // Process a block. For this example, we are going to intentionally // cause an error by trying to process the genesis block which already // exists. isOrphan, err := chain.ProcessBlock(genesisBlock, btcchain.BFNone) if err != nil { fmt.Printf("Failed to process block: %v\n", err) return } fmt.Printf("Block accepted. Is it an orphan?: %v", isOrphan) // Output: // Failed to process block: already have block 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f }
// TestCreateOpenFail ensures that errors which occur while opening or closing // a database are handled properly. func TestCreateOpenFail(t *testing.T) { // bogusCreateDB is a function which acts as a bogus create and open // driver function that intentionally returns a failure which can be // detected. dbType := "createopenfail" openError := fmt.Errorf("failed to create or open database for "+ "database type [%v]", dbType) bogusCreateDB := func(args ...interface{}) (btcdb.Db, error) { return nil, openError } // Create and add driver that intentionally fails when created or opened // to ensure errors on database open and create are handled properly. driver := btcdb.DriverDB{ DbType: dbType, CreateDB: bogusCreateDB, OpenDB: bogusCreateDB, } btcdb.AddDBDriver(driver) // Ensure creating a database with the new type fails with the expected // error. _, err := btcdb.CreateDB(dbType, "createfailtest") if err != openError { t.Errorf("TestCreateOpenFail: expected error not received - "+ "got: %v, want %v", err, openError) return } // Ensure opening a database with the new type fails with the expected // error. _, err = btcdb.OpenDB(dbType, "openfailtest") if err != openError { t.Errorf("TestCreateOpenFail: expected error not received - "+ "got: %v, want %v", err, openError) return } }
// loadBlockDB opens the block database and returns a handle to it. func loadBlockDB() (btcdb.Db, error) { // The database name is based on the database type. dbName := blockDbNamePrefix + "_" + cfg.DbType if cfg.DbType == "sqlite" { dbName = dbName + ".db" } dbPath := filepath.Join(cfg.DataDir, dbName) log.Infof("Loading block database from '%s'", dbPath) db, err := btcdb.OpenDB(cfg.DbType, dbPath) if err != nil { // Return the error if it's not because the database doesn't // exist. if err != btcdb.ErrDbDoesNotExist { return nil, err } // Create the db if it does not exist. err = os.MkdirAll(cfg.DataDir, 0700) if err != nil { return nil, err } db, err = btcdb.CreateDB(cfg.DbType, dbPath) if err != nil { return nil, err } } // Get the latest block height from the database. _, height, err := db.NewestSha() if err != nil { db.Close() return nil, err } log.Infof("Block database loaded with block height %d", height) return db, nil }
// chainSetup is used to create a new db and chain instance with the genesis // block already inserted. In addition to the new chain instnce, it returns // a teardown function the caller should invoke when done testing to clean up. func chainSetup(dbName string) (*btcchain.BlockChain, func(), error) { if !isSupportedDbType(testDbType) { return nil, nil, fmt.Errorf("unsupported db type %v", testDbType) } // Handle memory database specially since it doesn't need the disk // specific handling. var db btcdb.Db var teardown func() if testDbType == "memdb" { ndb, err := btcdb.CreateDB(testDbType) if err != nil { return nil, nil, fmt.Errorf("error creating db: %v", err) } db = ndb // Setup a teardown function for cleaning up. This function is // returned to the caller to be invoked when it is done testing. teardown = func() { db.Close() } } else { // Create the root directory for test databases. if !fileExists(testDbRoot) { if err := os.MkdirAll(testDbRoot, 0700); err != nil { err := fmt.Errorf("unable to create test db "+ "root: %v", err) return nil, nil, err } } // Create a new database to store the accepted blocks into. dbPath := filepath.Join(testDbRoot, dbName) _ = os.RemoveAll(dbPath) ndb, err := btcdb.CreateDB(testDbType, dbPath) if err != nil { return nil, nil, fmt.Errorf("error creating db: %v", err) } db = ndb // Setup a teardown function for cleaning up. This function is // returned to the caller to be invoked when it is done testing. teardown = func() { dbVersionPath := filepath.Join(testDbRoot, dbName+".ver") db.Sync() db.Close() os.RemoveAll(dbPath) os.Remove(dbVersionPath) os.RemoveAll(testDbRoot) } } // Insert the main network genesis block. This is part of the initial // database setup. genesisBlock := btcutil.NewBlock(&btcwire.GenesisBlock) _, err := db.InsertBlock(genesisBlock) if err != nil { teardown() err := fmt.Errorf("failed to insert genesis block: %v", err) return nil, nil, err } chain := btcchain.New(db, btcwire.MainNet, nil) return chain, teardown, nil }
func testBackout(t *testing.T, mode int) { // simplified basic operation is: // 1) fetch block from remote server // 2) look up all txin (except coinbase in db) // 3) insert block // Ignore db remove errors since it means we didn't have an old one. dbname := "tstdbop2" _ = os.Remove(dbname) db, err := btcdb.CreateDB("sqlite", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer os.Remove(dbname) defer db.Close() switch mode { case dbTmDefault: // default // no setup case dbTmNormal: // explicit normal db.SetDBInsertMode(btcdb.InsertNormal) case dbTmFast: // fast mode db.SetDBInsertMode(btcdb.InsertFast) if sqldb, ok := db.(*sqlite3.SqliteDb); ok { sqldb.TempTblMax = 100 } else { t.Errorf("not right type") } } // Since we are dealing with small dataset, reduce cache size sqlite3.SetBlockCacheSize(db, 2) sqlite3.SetTxCacheSize(db, 3) testdatafile := filepath.Join("testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if len(blocks) < 120 { t.Errorf("test data too small") return } err = nil for height := int64(1); height < int64(len(blocks)); height++ { if height == 100 { t.Logf("Syncing at block height 100") db.Sync() } if height == 120 { t.Logf("Simulating unexpected application quit") // Simulate unexpected application quit db.RollbackClose() break } block := blocks[height] newheight, err := db.InsertBlock(block) if err != nil { t.Errorf("failed to insert block %v err %v", height, err) break } if newheight != height { t.Errorf("height mismatch expect %v returned %v", height, newheight) break } } // db was closed at height 120, so no cleanup is possible. // reopen db db, err = btcdb.OpenDB("sqlite", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer db.Close() sha, err := blocks[99].Sha() if err != nil { t.Errorf("failed to get block 99 sha err %v", err) return } _ = db.ExistsSha(sha) _, err = db.FetchBlockBySha(sha) if err != nil { t.Errorf("failed to load block 99 from db", err) } sha, err = blocks[110].Sha() if err != nil { t.Errorf("failed to get block 110 sha err %v", err) return } _ = db.ExistsSha(sha) _, err = db.FetchBlockBySha(sha) if err == nil { t.Errorf("loaded block 110 from db, failure expected") return } block := blocks[110] mblock := block.MsgBlock() txsha, err := mblock.Transactions[0].TxSha(block.ProtocolVersion()) _, _, _, err = db.FetchTxBySha(&txsha) _, err = db.FetchTxUsedBySha(&txsha) block = blocks[99] mblock = block.MsgBlock() txsha, err = mblock.Transactions[0].TxSha(block.ProtocolVersion()) oldused, err := db.FetchTxUsedBySha(&txsha) err = db.InsertTx(&txsha, 99, 1024, 1048, oldused) if err == nil { t.Errorf("dup insert of tx succeeded") return } }
// insert every block in the test chain // after each insert, fetch all the tx affected by the latest // block and verify that the the tx is spent/unspent // new tx should be fully unspent, referenced tx should have // the associated txout set to spent. func testUnspentInsert(t *testing.T) { // Ignore db remove errors since it means we didn't have an old one. dbname := fmt.Sprintf("tstdbuspnt1") dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) _ = os.RemoveAll(dbnamever) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer os.RemoveAll(dbname) defer os.RemoveAll(dbnamever) defer db.Close() blocks := loadblocks(t) endtest: for height := int64(0); height < int64(len(blocks)); height++ { block := blocks[height] // look up inputs to this x mblock := block.MsgBlock() var txneededList []*btcwire.ShaHash var txlookupList []*btcwire.ShaHash var txOutList []*btcwire.ShaHash var txInList []*btcwire.OutPoint for _, tx := range mblock.Transactions { for _, txin := range tx.TxIn { if txin.PreviousOutpoint.Index == uint32(4294967295) { continue } origintxsha := &txin.PreviousOutpoint.Hash txInList = append(txInList, &txin.PreviousOutpoint) txneededList = append(txneededList, origintxsha) txlookupList = append(txlookupList, origintxsha) if !db.ExistsTxSha(origintxsha) { t.Errorf("referenced tx not found %v ", origintxsha) } } txshaname, _ := tx.TxSha() txlookupList = append(txlookupList, &txshaname) txOutList = append(txOutList, &txshaname) } txneededmap := map[btcwire.ShaHash]*btcdb.TxListReply{} txlist := db.FetchUnSpentTxByShaList(txneededList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) break endtest } txneededmap[*txe.Sha] = txe } for _, spend := range txInList { itxe := txneededmap[spend.Hash] if itxe.TxSpent[spend.Index] == true { t.Errorf("txin %v:%v is already spent", spend.Hash, spend.Index) } } newheight, err := db.InsertBlock(block) if err != nil { t.Errorf("failed to insert block %v err %v", height, err) break endtest } if newheight != height { t.Errorf("height mismatch expect %v returned %v", height, newheight) break endtest } txlookupmap := map[btcwire.ShaHash]*btcdb.TxListReply{} txlist = db.FetchTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) break endtest } txlookupmap[*txe.Sha] = txe } for _, spend := range txInList { itxe := txlookupmap[spend.Hash] if itxe.TxSpent[spend.Index] == false { t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) } } for _, txo := range txOutList { itxe := txlookupmap[*txo] for i, spent := range itxe.TxSpent { if spent == true { t.Errorf("freshly inserted tx %v already spent %v", txo, i) } } } if len(txInList) == 0 { continue } dropblock := blocks[height-1] dropsha, _ := dropblock.Sha() err = db.DropAfterBlockBySha(dropsha) if err != nil { t.Errorf("failed to drop block %v err %v", height, err) break endtest } txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} txlist = db.FetchUnSpentTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { if _, ok := txneededmap[*txe.Sha]; ok { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) break endtest } } txlookupmap[*txe.Sha] = txe } for _, spend := range txInList { itxe := txlookupmap[spend.Hash] if itxe.TxSpent[spend.Index] == true { t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) } } newheight, err = db.InsertBlock(block) if err != nil { t.Errorf("failed to insert block %v err %v", height, err) break endtest } txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} txlist = db.FetchTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) break endtest } txlookupmap[*txe.Sha] = txe } for _, spend := range txInList { itxe := txlookupmap[spend.Hash] if itxe.TxSpent[spend.Index] == false { t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) } } } }
func main() { cfg := config{ DbType: "leveldb", DataDir: filepath.Join(btcdHomeDir(), "data"), } parser := flags.NewParser(&cfg, flags.Default) _, err := parser.Parse() if err != nil { if e, ok := err.(*flags.Error); !ok || e.Type != flags.ErrHelp { parser.WriteHelp(os.Stderr) } return } runtime.GOMAXPROCS(runtime.NumCPU()) log, err = seelog.LoggerFromWriterWithMinLevel(os.Stdout, seelog.InfoLvl) if err != nil { fmt.Fprintf(os.Stderr, "failed to create logger: %v", err) return } defer log.Flush() btcdb.UseLogger(log) var testnet string if cfg.TestNet3 { testnet = "testnet" } else { testnet = "mainnet" } cfg.DataDir = filepath.Join(cfg.DataDir, testnet) err = os.MkdirAll(cfg.DataDir, 0700) if err != nil { fmt.Printf("unable to create db repo area %v, %v", cfg.DataDir, err) } blockDbNamePrefix := "blocks" dbName := blockDbNamePrefix + "_" + cfg.DbType if cfg.DbType == "sqlite" { dbName = dbName + ".db" } dbPath := filepath.Join(cfg.DataDir, dbName) log.Infof("loading db") db, err := btcdb.CreateDB(cfg.DbType, dbPath) if err != nil { log.Warnf("db open failed: %v", err) return } defer db.Close() log.Infof("db created") var fi io.ReadCloser fi, err = os.Open(cfg.InFile) if err != nil { log.Warnf("failed to open file %v, err %v", cfg.InFile, err) } defer func() { if err := fi.Close(); err != nil { log.Warn("failed to close file %v %v", cfg.InFile, err) } }() bufqueue := make(chan *bufQueue, 2) blkqueue := make(chan *blkQueue, 2) for i := 0; i < runtime.NumCPU(); i++ { go processBuf(i, bufqueue, blkqueue) } go processBuf(0, bufqueue, blkqueue) go readBlocks(fi, bufqueue) var eheight int64 doneMap := map[int64]*blkQueue{} for { select { case blkM := <-blkqueue: doneMap[blkM.height] = blkM for { if blkP, ok := doneMap[eheight]; ok { delete(doneMap, eheight) blkP.complete <- true db.InsertBlock(blkP.blk) if cfg.Progress && eheight%int64(1) == 0 { log.Infof("Processing block %v", eheight) } eheight++ if eheight%2000 == 0 { f, err := os.Create(fmt.Sprintf("profile.%d", eheight)) if err == nil { pprof.WriteHeapProfile(f) f.Close() } else { log.Warnf("profile failed %v", err) } } } else { break } } } } }
func main() { cfg := config{ DbType: "leveldb", DataDir: defaultDataDir, } parser := flags.NewParser(&cfg, flags.Default) _, err := parser.Parse() if err != nil { if e, ok := err.(*flags.Error); !ok || e.Type != flags.ErrHelp { parser.WriteHelp(os.Stderr) } return } // Use all processor cores. runtime.GOMAXPROCS(runtime.NumCPU()) // Up some limits. if err := limits.SetLimits(); err != nil { os.Exit(1) } backendLogger := btclog.NewDefaultBackendLogger() defer backendLogger.Flush() log = btclog.NewSubsystemLogger(backendLogger, "") btcdb.UseLogger(log) var testnet string if cfg.TestNet3 { testnet = "testnet" } else { testnet = "mainnet" } cfg.DataDir = filepath.Join(cfg.DataDir, testnet) err = os.MkdirAll(cfg.DataDir, 0700) if err != nil { fmt.Printf("unable to create db repo area %v, %v", cfg.DataDir, err) } blockDbNamePrefix := "blocks" dbName := blockDbNamePrefix + "_" + cfg.DbType if cfg.DbType == "sqlite" { dbName = dbName + ".db" } dbPath := filepath.Join(cfg.DataDir, dbName) log.Infof("loading db") db, err := btcdb.CreateDB(cfg.DbType, dbPath) if err != nil { log.Warnf("db open failed: %v", err) return } defer db.Close() log.Infof("db created") var fi io.ReadCloser fi, err = os.Open(cfg.InFile) if err != nil { log.Warnf("failed to open file %v, err %v", cfg.InFile, err) } defer func() { if err := fi.Close(); err != nil { log.Warn("failed to close file %v %v", cfg.InFile, err) } }() bufqueue := make(chan *bufQueue, 2) blkqueue := make(chan *blkQueue, 2) for i := 0; i < runtime.NumCPU(); i++ { go processBuf(i, bufqueue, blkqueue) } go processBuf(0, bufqueue, blkqueue) go readBlocks(fi, bufqueue) var eheight int64 doneMap := map[int64]*blkQueue{} for { select { case blkM := <-blkqueue: doneMap[blkM.height] = blkM for { if blkP, ok := doneMap[eheight]; ok { delete(doneMap, eheight) blkP.complete <- true db.InsertBlock(blkP.blk) if cfg.Progress && eheight%int64(10000) == 0 { log.Infof("Processing block %v", eheight) } eheight++ } else { break } } } } if cfg.Progress { log.Infof("Processing block %v", eheight) } }
func testOperationalMode(t *testing.T, mode int) { // simplified basic operation is: // 1) fetch block from remote server // 2) look up all txin (except coinbase in db) // 3) insert block // Ignore db remove errors since it means we didn't have an old one. dbname := "tstdbop1" _ = os.Remove(dbname) db, err := btcdb.CreateDB("sqlite", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer os.Remove(dbname) defer db.Close() switch mode { case dbTmDefault: // default // no setup case dbTmNormal: // explicit normal db.SetDBInsertMode(btcdb.InsertNormal) case dbTmFast: // fast mode db.SetDBInsertMode(btcdb.InsertFast) if sqldb, ok := db.(*sqlite3.SqliteDb); ok { sqldb.TempTblMax = 100 } else { t.Errorf("not right type") } case dbTmNoVerify: // validated block db.SetDBInsertMode(btcdb.InsertValidatedInput) } // Since we are dealing with small dataset, reduce cache size sqlite3.SetBlockCacheSize(db, 2) sqlite3.SetTxCacheSize(db, 3) testdatafile := filepath.Join("testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if err != nil { t.Errorf("Unable to load blocks from test data for mode %v: %v", mode, err) return } err = nil out: for height := int64(1); height < int64(len(blocks)); height++ { block := blocks[height] if mode != dbTmNoVerify { // except for NoVerify which does not allow lookups check inputs mblock := block.MsgBlock() var txneededList []*btcwire.ShaHash for _, tx := range mblock.Transactions { for _, txin := range tx.TxIn { if txin.PreviousOutpoint.Index == uint32(4294967295) { continue } origintxsha := &txin.PreviousOutpoint.Hash txneededList = append(txneededList, origintxsha) _, _, _, _, err := db.FetchTxAllBySha(origintxsha) if err != nil { t.Errorf("referenced tx not found %v err %v ", origintxsha, err) } _, _, _, _, err = db.FetchTxAllBySha(origintxsha) if err != nil { t.Errorf("referenced tx not found %v err %v ", origintxsha, err) } _, _, _, err = db.FetchTxBySha(origintxsha) if err != nil { t.Errorf("referenced tx not found %v err %v ", origintxsha, err) } _, _, err = db.FetchTxBufBySha(origintxsha) if err != nil { t.Errorf("referenced tx not found %v err %v ", origintxsha, err) } _, err = db.FetchTxUsedBySha(origintxsha) if err != nil { t.Errorf("tx used fetch fail %v err %v ", origintxsha, err) } } } txlist := db.FetchTxByShaList(txneededList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) break out } } } newheight, err := db.InsertBlock(block) if err != nil { t.Errorf("failed to insert block %v err %v", height, err) break out } if newheight != height { t.Errorf("height mismatch expect %v returned %v", height, newheight) break out } } switch mode { case dbTmDefault: // default // no cleanup case dbTmNormal: // explicit normal // no cleanup case dbTmFast: // fast mode db.SetDBInsertMode(btcdb.InsertNormal) case dbTmNoVerify: // validated block db.SetDBInsertMode(btcdb.InsertNormal) } }
func main() { var err error var dbType string var datadir string var infile string var progress int flag.StringVar(&dbType, "dbtype", "", "Database backend to use for the Block Chain") flag.StringVar(&datadir, "datadir", "", "Directory to store data") flag.StringVar(&infile, "i", "", "infile") flag.IntVar(&progress, "p", 0, "show progress") flag.Parse() runtime.GOMAXPROCS(runtime.NumCPU()) if len(infile) == 0 { fmt.Printf("Must specify inputfile") return } log, err = seelog.LoggerFromWriterWithMinLevel(os.Stdout, seelog.InfoLvl) if err != nil { fmt.Fprintf(os.Stderr, "failed to create logger: %v", err) return } defer log.Flush() btcdb.UseLogger(log) if len(dbType) == 0 { dbType = "sqlite" } if len(datadir) == 0 { datadir = filepath.Join(btcdHomeDir(), "data") } datadir = filepath.Join(datadir, "mainnet") err = os.MkdirAll(datadir, 0700) if err != nil { fmt.Printf("unable to create db repo area %v, %v", datadir, err) } blockDbNamePrefix := "blocks" dbName := blockDbNamePrefix + "_" + dbType if dbType == "sqlite" { dbName = dbName + ".db" } dbPath := filepath.Join(datadir, dbName) log.Infof("loading db") db, err := btcdb.CreateDB(dbType, dbPath) if err != nil { log.Warnf("db open failed: %v", err) return } defer db.Close() log.Infof("db created") var fi io.ReadCloser fi, err = os.Open(infile) if err != nil { log.Warnf("failed to open file %v, err %v", infile, err) } defer func() { if err := fi.Close(); err != nil { log.Warn("failed to close file %v %v", infile, err) } }() bufqueue := make(chan *bufQueue, 2) blkqueue := make(chan *blkQueue, 2) for i := 0; i < runtime.NumCPU(); i++ { go processBuf(i, bufqueue, blkqueue) } go processBuf(0, bufqueue, blkqueue) go readBlocks(fi, bufqueue) var eheight int64 doneMap := map[int64]*blkQueue{} for { select { case blkM := <-blkqueue: doneMap[blkM.height] = blkM for { if blkP, ok := doneMap[eheight]; ok { delete(doneMap, eheight) blkP.complete <- true db.InsertBlock(blkP.blk) if progress != 0 && eheight%int64(progress) == 0 { log.Infof("Processing block %v", eheight) } eheight++ if eheight%2000 == 0 { f, err := os.Create(fmt.Sprintf("profile.%d", eheight)) if err == nil { pprof.WriteHeapProfile(f) f.Close() } else { log.Warnf("profile failed %v", err) } } } else { break } } } } }
func testBackout(t *testing.T) { // simplified basic operation is: // 1) fetch block from remote server // 2) look up all txin (except coinbase in db) // 3) insert block // Ignore db remove errors since it means we didn't have an old one. dbname := fmt.Sprintf("tstdbop2") dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) _ = os.RemoveAll(dbnamever) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer os.RemoveAll(dbname) defer os.RemoveAll(dbnamever) defer db.Close() testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if len(blocks) < 120 { t.Errorf("test data too small") return } err = nil for height := int64(0); height < int64(len(blocks)); height++ { if height == 100 { t.Logf("Syncing at block height 100") db.Sync() } if height == 120 { t.Logf("Simulating unexpected application quit") // Simulate unexpected application quit db.RollbackClose() break } block := blocks[height] newheight, err := db.InsertBlock(block) if err != nil { t.Errorf("failed to insert block %v err %v", height, err) return } if newheight != height { t.Errorf("height mismatch expect %v returned %v", height, newheight) return } } // db was closed at height 120, so no cleanup is possible. // reopen db db, err = btcdb.OpenDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer func() { if err := db.Close(); err != nil { t.Errorf("Close: unexpected error: %v", err) } }() sha, err := blocks[99].Sha() if err != nil { t.Errorf("failed to get block 99 sha err %v", err) return } if _, err := db.ExistsSha(sha); err != nil { t.Errorf("ExistsSha: unexpected error: %v") } _, err = db.FetchBlockBySha(sha) if err != nil { t.Errorf("failed to load block 99 from db %v", err) return } sha, err = blocks[119].Sha() if err != nil { t.Errorf("failed to get block 110 sha err %v", err) return } if _, err := db.ExistsSha(sha); err != nil { t.Errorf("ExistsSha: unexpected error: %v") } _, err = db.FetchBlockBySha(sha) if err != nil { t.Errorf("loaded block 119 from db") return } block := blocks[119] mblock := block.MsgBlock() txsha, err := mblock.Transactions[0].TxSha() exists, err := db.ExistsTxSha(&txsha) if err != nil { t.Errorf("ExistsTxSha: unexpected error %v ", err) } if !exists { t.Errorf("tx %v not located db\n", txsha) } _, err = db.FetchTxBySha(&txsha) if err != nil { t.Errorf("tx %v not located db\n", txsha) return } }
// TestClosed ensure calling the interface functions on a closed database // returns appropriate errors for the interface functions that return errors // and does not panic or otherwise misbehave for functions which do not return // errors. func TestClosed(t *testing.T) { db, err := btcdb.CreateDB("memdb") if err != nil { t.Errorf("Failed to open test database %v", err) return } _, err = db.InsertBlock(btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock)) if err != nil { t.Errorf("InsertBlock: %v", err) } if err := db.Close(); err != nil { t.Errorf("Close: unexpected error %v", err) } genesisHash := btcnet.MainNetParams.GenesisHash if err := db.DropAfterBlockBySha(genesisHash); err != memdb.ErrDbClosed { t.Errorf("DropAfterBlockBySha: unexpected error %v", err) } if _, err := db.ExistsSha(genesisHash); err != memdb.ErrDbClosed { t.Errorf("ExistsSha: Unexpected error: %v", err) } if _, err := db.FetchBlockBySha(genesisHash); err != memdb.ErrDbClosed { t.Errorf("FetchBlockBySha: unexpected error %v", err) } if _, err := db.FetchBlockShaByHeight(0); err != memdb.ErrDbClosed { t.Errorf("FetchBlockShaByHeight: unexpected error %v", err) } if _, err := db.FetchHeightRange(0, 1); err != memdb.ErrDbClosed { t.Errorf("FetchHeightRange: unexpected error %v", err) } genesisCoinbaseTx := btcnet.MainNetParams.GenesisBlock.Transactions[0] coinbaseHash, err := genesisCoinbaseTx.TxSha() if err != nil { t.Errorf("TxSha: unexpected error %v", err) } if _, err := db.ExistsTxSha(&coinbaseHash); err != memdb.ErrDbClosed { t.Errorf("ExistsTxSha: unexpected error %v", err) } if _, err := db.FetchTxBySha(genesisHash); err != memdb.ErrDbClosed { t.Errorf("FetchTxBySha: unexpected error %v", err) } requestHashes := []*btcwire.ShaHash{genesisHash} reply := db.FetchTxByShaList(requestHashes) if len(reply) != len(requestHashes) { t.Errorf("FetchUnSpentTxByShaList unexpected number of replies "+ "got: %d, want: %d", len(reply), len(requestHashes)) } for i, txLR := range reply { wantReply := &btcdb.TxListReply{ Sha: requestHashes[i], Err: memdb.ErrDbClosed, } if !reflect.DeepEqual(wantReply, txLR) { t.Errorf("FetchTxByShaList unexpected reply\ngot: %v\n"+ "want: %v", txLR, wantReply) } } reply = db.FetchUnSpentTxByShaList(requestHashes) if len(reply) != len(requestHashes) { t.Errorf("FetchUnSpentTxByShaList unexpected number of replies "+ "got: %d, want: %d", len(reply), len(requestHashes)) } for i, txLR := range reply { wantReply := &btcdb.TxListReply{ Sha: requestHashes[i], Err: memdb.ErrDbClosed, } if !reflect.DeepEqual(wantReply, txLR) { t.Errorf("FetchUnSpentTxByShaList unexpected reply\n"+ "got: %v\nwant: %v", txLR, wantReply) } } if _, _, err := db.NewestSha(); err != memdb.ErrDbClosed { t.Errorf("NewestSha: unexpected error %v", err) } if err := db.Sync(); err != memdb.ErrDbClosed { t.Errorf("Sync: unexpected error %v", err) } if err := db.RollbackClose(); err != memdb.ErrDbClosed { t.Errorf("RollbackClose: unexpected error %v", err) } if err := db.Close(); err != memdb.ErrDbClosed { t.Errorf("Close: unexpected error %v", err) } }
func testOperationalMode(t *testing.T) { // simplified basic operation is: // 1) fetch block from remote server // 2) look up all txin (except coinbase in db) // 3) insert block // Ignore db remove errors since it means we didn't have an old one. dbname := fmt.Sprintf("tstdbop1") dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) _ = os.RemoveAll(dbnamever) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer os.RemoveAll(dbname) defer os.RemoveAll(dbnamever) defer func() { if err := db.Close(); err != nil { t.Errorf("Close: unexpected error: %v", err) } }() testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if err != nil { t.Errorf("Unable to load blocks from test data: %v", err) return } err = nil out: for height := int64(0); height < int64(len(blocks)); height++ { block := blocks[height] mblock := block.MsgBlock() var txneededList []*btcwire.ShaHash for _, tx := range mblock.Transactions { for _, txin := range tx.TxIn { if txin.PreviousOutPoint.Index == uint32(4294967295) { continue } origintxsha := &txin.PreviousOutPoint.Hash txneededList = append(txneededList, origintxsha) exists, err := db.ExistsTxSha(origintxsha) if err != nil { t.Errorf("ExistsTxSha: unexpected error %v ", err) } if !exists { t.Errorf("referenced tx not found %v ", origintxsha) } _, err = db.FetchTxBySha(origintxsha) if err != nil { t.Errorf("referenced tx not found %v err %v ", origintxsha, err) } } } txlist := db.FetchUnSpentTxByShaList(txneededList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) break out } } newheight, err := db.InsertBlock(block) if err != nil { t.Errorf("failed to insert block %v err %v", height, err) break out } if newheight != height { t.Errorf("height mismatch expect %v returned %v", height, newheight) break out } newSha, blkid, err := db.NewestSha() if err != nil { t.Errorf("failed to obtain latest sha %v %v", height, err) } if blkid != height { t.Errorf("height doe not match latest block height %v %v %v", blkid, height, err) } blkSha, _ := block.Sha() if *newSha != *blkSha { t.Errorf("Newest block sha does not match freshly inserted one %v %v %v ", newSha, blkSha, err) } } // now that db is populated, do some additional test testFetchRangeHeight(t, db, blocks) }
func Test_dupTx(t *testing.T) { // Ignore db remove errors since it means we didn't have an old one. dbname := fmt.Sprintf("tstdbdup0") dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) _ = os.RemoveAll(dbnamever) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer os.RemoveAll(dbname) defer os.RemoveAll(dbnamever) defer db.Close() testdatafile := filepath.Join("testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if err != nil { t.Errorf("Unable to load blocks from test data for: %v", err) return } var lastSha *btcwire.ShaHash // Populate with the fisrt 256 blocks, so we have blocks to 'mess with' err = nil out: for height := int64(0); height < int64(len(blocks)); height++ { block := blocks[height] // except for NoVerify which does not allow lookups check inputs mblock := block.MsgBlock() var txneededList []*btcwire.ShaHash for _, tx := range mblock.Transactions { for _, txin := range tx.TxIn { if txin.PreviousOutpoint.Index == uint32(4294967295) { continue } origintxsha := &txin.PreviousOutpoint.Hash txneededList = append(txneededList, origintxsha) if !db.ExistsTxSha(origintxsha) { t.Errorf("referenced tx not found %v ", origintxsha) } _, err = db.FetchTxBySha(origintxsha) if err != nil { t.Errorf("referenced tx not found %v err %v ", origintxsha, err) } } } txlist := db.FetchUnSpentTxByShaList(txneededList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) break out } } newheight, err := db.InsertBlock(block) if err != nil { t.Errorf("failed to insert block %v err %v", height, err) break out } if newheight != height { t.Errorf("height mismatch expect %v returned %v", height, newheight) break out } newSha, blkid, err := db.NewestSha() if err != nil { t.Errorf("failed to obtain latest sha %v %v", height, err) } if blkid != height { t.Errorf("height doe not match latest block height %v %v %v", blkid, height, err) } blkSha, _ := block.Sha() if *newSha != *blkSha { t.Errorf("Newest block sha does not match freshly inserted one %v %v %v ", newSha, blkSha, err) } lastSha = blkSha } // genrate a new block based on the last sha // these block are not verified, so there are a bunch of garbage fields // in the 'generated' block. var bh btcwire.BlockHeader bh.Version = 2 bh.PrevBlock = *lastSha // Bits, Nonce are not filled in mblk := btcwire.NewMsgBlock(&bh) hash, _ := btcwire.NewShaHashFromStr("df2b060fa2e5e9c8ed5eaf6a45c13753ec8c63282b2688322eba40cd98ea067a") po := btcwire.NewOutPoint(hash, 0) txI := btcwire.NewTxIn(po, []byte("garbage")) txO := btcwire.NewTxOut(50000000, []byte("garbageout")) var tx btcwire.MsgTx tx.AddTxIn(txI) tx.AddTxOut(txO) mblk.AddTransaction(&tx) blk := btcutil.NewBlock(mblk) fetchList := []*btcwire.ShaHash{hash} listReply := db.FetchUnSpentTxByShaList(fetchList) for _, lr := range listReply { if lr.Err != nil { t.Errorf("sha %v spent %v err %v\n", lr.Sha, lr.TxSpent, lr.Err) } } _, err = db.InsertBlock(blk) if err != nil { t.Errorf("failed to insert phony block %v", err) } // ok, did it 'spend' the tx ? listReply = db.FetchUnSpentTxByShaList(fetchList) for _, lr := range listReply { if lr.Err != btcdb.TxShaMissing { t.Errorf("sha %v spent %v err %v\n", lr.Sha, lr.TxSpent, lr.Err) } } txshalist, _ := blk.TxShas() for _, txsha := range txshalist { txReply, err := db.FetchTxBySha(txsha) if err != nil { t.Errorf("fully spent lookup %v err %v\n", hash, err) } else { for _, lr := range txReply { if lr.Err != nil { fmt.Errorf("stx %v spent %v err %v\n", lr.Sha, lr.TxSpent, lr.Err) } } } } t.Logf("Dropping block") err = db.DropAfterBlockBySha(lastSha) if err != nil { t.Errorf("failed to drop spending block %v", err) } }
// TestClosed ensure calling the interface functions on a closed database // returns appropriate errors for the interface functions that return errors // and does not panic or otherwise misbehave for functions which do not return // errors. func TestClosed(t *testing.T) { db, err := btcdb.CreateDB("memdb") if err != nil { t.Errorf("Failed to open test database %v", err) return } _, err = db.InsertBlock(btcutil.NewBlock(&btcwire.GenesisBlock)) if err != nil { t.Errorf("InsertBlock: %v", err) } db.Close() genesisHash := &btcwire.GenesisHash if err := db.DropAfterBlockBySha(genesisHash); err != memdb.ErrDbClosed { t.Errorf("DropAfterBlockBySha: unexpected error %v", err) } if exists := db.ExistsSha(genesisHash); exists != false { t.Errorf("ExistsSha: genesis hash exists after close") } if _, err := db.FetchBlockBySha(genesisHash); err != memdb.ErrDbClosed { t.Errorf("FetchBlockBySha: unexpected error %v", err) } if _, err := db.FetchBlockShaByHeight(0); err != memdb.ErrDbClosed { t.Errorf("FetchBlockShaByHeight: unexpected error %v", err) } if _, err := db.FetchHeightRange(0, 1); err != memdb.ErrDbClosed { t.Errorf("FetchHeightRange: unexpected error %v", err) } genesisMerkleRoot := &btcwire.GenesisMerkleRoot if exists := db.ExistsTxSha(genesisMerkleRoot); exists != false { t.Errorf("ExistsTxSha: hash %v exists when it shouldn't", genesisMerkleRoot) } if _, err := db.FetchTxBySha(genesisHash); err != memdb.ErrDbClosed { t.Errorf("FetchTxBySha: unexpected error %v", err) } requestHashes := []*btcwire.ShaHash{genesisHash} reply := db.FetchTxByShaList(requestHashes) if len(reply) != len(requestHashes) { t.Errorf("FetchUnSpentTxByShaList unexpected number of replies "+ "got: %d, want: %d", len(reply), len(requestHashes)) } for i, txLR := range reply { wantReply := &btcdb.TxListReply{ Sha: requestHashes[i], Err: memdb.ErrDbClosed, } if !reflect.DeepEqual(wantReply, txLR) { t.Errorf("FetchTxByShaList unexpected reply\ngot: %v\n"+ "want: %v", txLR, wantReply) } } reply = db.FetchUnSpentTxByShaList(requestHashes) if len(reply) != len(requestHashes) { t.Errorf("FetchUnSpentTxByShaList unexpected number of replies "+ "got: %d, want: %d", len(reply), len(requestHashes)) } for i, txLR := range reply { wantReply := &btcdb.TxListReply{ Sha: requestHashes[i], Err: memdb.ErrDbClosed, } if !reflect.DeepEqual(wantReply, txLR) { t.Errorf("FetchUnSpentTxByShaList unexpected reply\n"+ "got: %v\nwant: %v", txLR, wantReply) } } if _, _, err := db.NewestSha(); err != memdb.ErrDbClosed { t.Errorf("NewestSha: unexpected error %v", err) } // The following calls don't return errors from the interface to be able // to detect a closed database, so just call them to ensure there are no // panics. db.Sync() db.RollbackClose() }