// loadBlocks reads files containing bitcoin block data (gzipped but otherwise // in the format bitcoind writes) from disk and returns them as an array of // btcutil.Block. This is largely borrowed from the test code in btcdb. func loadBlocks(filename string) (blocks []*btcutil.Block, err error) { filename = filepath.Join("testdata/", filename) var network = wire.MainNet var dr io.Reader var fi io.ReadCloser fi, err = os.Open(filename) if err != nil { return } if strings.HasSuffix(filename, ".bz2") { dr = bzip2.NewReader(fi) } else { dr = fi } defer fi.Close() var block *btcutil.Block err = nil for height := int64(1); err == nil; height++ { var rintbuf uint32 err = binary.Read(dr, binary.LittleEndian, &rintbuf) if err == io.EOF { // hit end of file at expected offset: no warning height-- err = nil break } if err != nil { break } if rintbuf != uint32(network) { break } err = binary.Read(dr, binary.LittleEndian, &rintbuf) blocklen := rintbuf rbytes := make([]byte, blocklen) // read block dr.Read(rbytes) block, err = btcutil.NewBlockFromBytes(rbytes) if err != nil { return } blocks = append(blocks, block) } return }
// processBlock potentially imports the block into the database. It first // deserializes the raw block while checking for errors. Already known blocks // are skipped and orphan blocks are considered errors. Finally, it runs the // block through the chain rules to ensure it follows all rules and matches // up to the known checkpoint. Returns whether the block was imported along // with any potential errors. func (bi *blockImporter) processBlock(serializedBlock []byte) (bool, error) { // Deserialize the block which includes checks for malformed blocks. block, err := btcutil.NewBlockFromBytes(serializedBlock) if err != nil { return false, err } // update progress statistics bi.lastBlockTime = block.MsgBlock().Header.Timestamp bi.receivedLogTx += int64(len(block.MsgBlock().Transactions)) // Skip blocks that already exist. blockHash := block.Hash() exists, err := bi.chain.HaveBlock(blockHash) if err != nil { return false, err } if exists { return false, nil } // Don't bother trying to process orphans. prevHash := &block.MsgBlock().Header.PrevBlock if !prevHash.IsEqual(&zeroHash) { exists, err := bi.chain.HaveBlock(prevHash) if err != nil { return false, err } if !exists { return false, fmt.Errorf("import file contains block "+ "%v which does not link to the available "+ "block chain", prevHash) } } // Ensure the blocks follows all of the chain rules and match up to the // known checkpoints. isMainChain, isOrphan, err := bi.chain.ProcessBlock(block, blockchain.BFFastAdd) if err != nil { return false, err } if !isMainChain { return false, fmt.Errorf("import file contains an block that "+ "does not extend the main chain: %v", blockHash) } if isOrphan { return false, fmt.Errorf("import file contains an orphan "+ "block: %v", blockHash) } return true, nil }
// processBlock potentially imports the block into the database. It first // deserializes the raw block while checking for errors. Already known blocks // are skipped and orphan blocks are considered errors. Returns whether the // block was imported along with any potential errors. // // NOTE: This is not a safe import as it does not verify chain rules. func (bi *blockImporter) processBlock(serializedBlock []byte) (bool, error) { // Deserialize the block which includes checks for malformed blocks. block, err := btcutil.NewBlockFromBytes(serializedBlock) if err != nil { return false, err } // update progress statistics bi.lastBlockTime = block.MsgBlock().Header.Timestamp bi.receivedLogTx += int64(len(block.MsgBlock().Transactions)) // Skip blocks that already exist. var exists bool err = bi.db.View(func(tx database.Tx) error { exists, err = tx.HasBlock(block.Hash()) return err }) if err != nil { return false, err } if exists { return false, nil } // Don't bother trying to process orphans. prevHash := &block.MsgBlock().Header.PrevBlock if !prevHash.IsEqual(&zeroHash) { var exists bool err := bi.db.View(func(tx database.Tx) error { exists, err = tx.HasBlock(prevHash) return err }) if err != nil { return false, err } if !exists { return false, fmt.Errorf("import file contains block "+ "%v which does not link to the available "+ "block chain", prevHash) } } // Put the blocks into the database with no checking of chain rules. err = bi.db.Update(func(tx database.Tx) error { return tx.StoreBlock(block) }) if err != nil { return false, err } return true, nil }
// loadBlocks loads the blocks contained in the testdata directory and returns // a slice of them. func loadBlocks(t *testing.T, dataFile string, network wire.BitcoinNet) ([]*btcutil.Block, error) { // Open the file that contains the blocks for reading. fi, err := os.Open(dataFile) if err != nil { t.Errorf("failed to open file %v, err %v", dataFile, err) return nil, err } defer func() { if err := fi.Close(); err != nil { t.Errorf("failed to close file %v %v", dataFile, err) } }() dr := bzip2.NewReader(fi) // Set the first block as the genesis block. blocks := make([]*btcutil.Block, 0, 256) genesis := btcutil.NewBlock(chaincfg.MainNetParams.GenesisBlock) blocks = append(blocks, genesis) // Load the remaining blocks. for height := 1; ; height++ { var net uint32 err := binary.Read(dr, binary.LittleEndian, &net) if err == io.EOF { // Hit end of file at the expected offset. No error. break } if err != nil { t.Errorf("Failed to load network type for block %d: %v", height, err) return nil, err } if net != uint32(network) { t.Errorf("Block doesn't match network: %v expects %v", net, network) return nil, err } var blockLen uint32 err = binary.Read(dr, binary.LittleEndian, &blockLen) if err != nil { t.Errorf("Failed to load block size for block %d: %v", height, err) return nil, err } // Read the block. blockBytes := make([]byte, blockLen) _, err = io.ReadFull(dr, blockBytes) if err != nil { t.Errorf("Failed to load block %d: %v", height, err) return nil, err } // Deserialize and store the block. block, err := btcutil.NewBlockFromBytes(blockBytes) if err != nil { t.Errorf("Failed to parse block %v: %v", height, err) return nil, err } blocks = append(blocks, block) } return blocks, nil }
// Init initializes the enabled indexes. This is called during chain // initialization and primarily consists of catching up all indexes to the // current best chain tip. This is necessary since each index can be disabled // and re-enabled at any time and attempting to catch-up indexes at the same // time new blocks are being downloaded would lead to an overall longer time to // catch up due to the I/O contention. // // This is part of the blockchain.IndexManager interface. func (m *Manager) Init(chain *blockchain.BlockChain) error { // Nothing to do when no indexes are enabled. if len(m.enabledIndexes) == 0 { return nil } // Finish and drops that were previously interrupted. if err := m.maybeFinishDrops(); err != nil { return err } // Create the initial state for the indexes as needed. err := m.db.Update(func(dbTx database.Tx) error { // Create the bucket for the current tips as needed. meta := dbTx.Metadata() _, err := meta.CreateBucketIfNotExists(indexTipsBucketName) if err != nil { return err } return m.maybeCreateIndexes(dbTx) }) if err != nil { return err } // Initialize each of the enabled indexes. for _, indexer := range m.enabledIndexes { if err := indexer.Init(); err != nil { return err } } // Rollback indexes to the main chain if their tip is an orphaned fork. // This is fairly unlikely, but it can happen if the chain is // reorganized while the index is disabled. This has to be done in // reverse order because later indexes can depend on earlier ones. for i := len(m.enabledIndexes); i > 0; i-- { indexer := m.enabledIndexes[i-1] // Fetch the current tip for the index. var height int32 var hash *chainhash.Hash err := m.db.View(func(dbTx database.Tx) error { idxKey := indexer.Key() hash, height, err = dbFetchIndexerTip(dbTx, idxKey) return err }) if err != nil { return err } // Nothing to do if the index does not have any entries yet. if height == -1 { continue } // Loop until the tip is a block that exists in the main chain. initialHeight := height for { exists, err := chain.MainChainHasBlock(hash) if err != nil { return err } if exists { break } // At this point the index tip is orphaned, so load the // orphaned block from the database directly and // disconnect it from the index. The block has to be // loaded directly since it is no longer in the main // chain and thus the chain.BlockByHash function would // error. err = m.db.Update(func(dbTx database.Tx) error { blockBytes, err := dbTx.FetchBlock(hash) if err != nil { return err } block, err := btcutil.NewBlockFromBytes(blockBytes) if err != nil { return err } block.SetHeight(height) // When the index requires all of the referenced // txouts they need to be retrieved from the // transaction index. var view *blockchain.UtxoViewpoint if indexNeedsInputs(indexer) { var err error view, err = makeUtxoView(dbTx, block) if err != nil { return err } } // Remove all of the index entries associated // with the block and update the indexer tip. err = dbIndexDisconnectBlock(dbTx, indexer, block, view) if err != nil { return err } // Update the tip to the previous block. hash = &block.MsgBlock().Header.PrevBlock height-- return nil }) if err != nil { return err } } if initialHeight != height { log.Infof("Removed %d orphaned blocks from %s "+ "(heights %d to %d)", initialHeight-height, indexer.Name(), height+1, initialHeight) } } // Fetch the current tip heights for each index along with tracking the // lowest one so the catchup code only needs to start at the earliest // block and is able to skip connecting the block for the indexes that // don't need it. bestHeight := chain.BestSnapshot().Height lowestHeight := bestHeight indexerHeights := make([]int32, len(m.enabledIndexes)) err = m.db.View(func(dbTx database.Tx) error { for i, indexer := range m.enabledIndexes { idxKey := indexer.Key() hash, height, err := dbFetchIndexerTip(dbTx, idxKey) if err != nil { return err } log.Debugf("Current %s tip (height %d, hash %v)", indexer.Name(), height, hash) indexerHeights[i] = height if height < lowestHeight { lowestHeight = height } } return nil }) if err != nil { return err } // Nothing to index if all of the indexes are caught up. if lowestHeight == bestHeight { return nil } // Create a progress logger for the indexing process below. progressLogger := newBlockProgressLogger("Indexed", log) // At this point, one or more indexes are behind the current best chain // tip and need to be caught up, so log the details and loop through // each block that needs to be indexed. log.Infof("Catching up indexes from height %d to %d", lowestHeight, bestHeight) for height := lowestHeight + 1; height <= bestHeight; height++ { // Load the block for the height since it is required to index // it. block, err := chain.BlockByHeight(height) if err != nil { return err } // Connect the block for all indexes that need it. var view *blockchain.UtxoViewpoint for i, indexer := range m.enabledIndexes { // Skip indexes that don't need to be updated with this // block. if indexerHeights[i] >= height { continue } err := m.db.Update(func(dbTx database.Tx) error { // When the index requires all of the referenced // txouts and they haven't been loaded yet, they // need to be retrieved from the transaction // index. if view == nil && indexNeedsInputs(indexer) { var err error view, err = makeUtxoView(dbTx, block) if err != nil { return err } } return dbIndexConnectBlock(dbTx, indexer, block, view) }) if err != nil { return err } indexerHeights[i] = height } // Log indexing progress. progressLogger.LogBlockHeight(block) } log.Infof("Indexes caught up to height %d", bestHeight) return nil }