// indexBlockAddrs returns a populated index of the all the transactions in the // passed block based on the addresses involved in each transaction. func (a *addrIndexer) indexBlockAddrs(blk *dcrutil.Block, parent *dcrutil.Block) (database.BlockAddrIndex, error) { var addrIndex database.BlockAddrIndex _, stxLocs, err := blk.TxLoc() if err != nil { return nil, err } txTreeRegularValid := dcrutil.IsFlagSet16(blk.MsgBlock().Header.VoteBits, dcrutil.BlockValid) // Add regular transactions iff the block was validated. if txTreeRegularValid { txLocs, _, err := parent.TxLoc() if err != nil { return nil, err } for txIdx, tx := range parent.Transactions() { // Tx's offset and length in the block. locInBlock := &txLocs[txIdx] // Coinbases don't have any inputs. if !blockchain.IsCoinBase(tx) { // Index the SPK's of each input's previous outpoint // transaction. for _, txIn := range tx.MsgTx().TxIn { prevOutTx, err := a.lookupTransaction( txIn.PreviousOutPoint.Hash, blk, parent) inputOutPoint := prevOutTx.TxOut[txIn.PreviousOutPoint.Index] toAppend, err := convertToAddrIndex(inputOutPoint.Version, inputOutPoint.PkScript, parent.Height(), locInBlock) if err != nil { adxrLog.Tracef("Error converting tx txin %v: %v", txIn.PreviousOutPoint.Hash, err) continue } addrIndex = append(addrIndex, toAppend...) } } for _, txOut := range tx.MsgTx().TxOut { toAppend, err := convertToAddrIndex(txOut.Version, txOut.PkScript, parent.Height(), locInBlock) if err != nil { adxrLog.Tracef("Error converting tx txout %v: %v", tx.MsgTx().TxSha(), err) continue } addrIndex = append(addrIndex, toAppend...) } } } // Add stake transactions. for stxIdx, stx := range blk.STransactions() { // Tx's offset and length in the block. locInBlock := &stxLocs[stxIdx] isSSGen, _ := stake.IsSSGen(stx) // Index the SPK's of each input's previous outpoint // transaction. for i, txIn := range stx.MsgTx().TxIn { // Stakebases don't have any inputs. if isSSGen && i == 0 { continue } // Lookup and fetch the referenced output's tx. prevOutTx, err := a.lookupTransaction( txIn.PreviousOutPoint.Hash, blk, parent) inputOutPoint := prevOutTx.TxOut[txIn.PreviousOutPoint.Index] toAppend, err := convertToAddrIndex(inputOutPoint.Version, inputOutPoint.PkScript, blk.Height(), locInBlock) if err != nil { adxrLog.Tracef("Error converting stx txin %v: %v", txIn.PreviousOutPoint.Hash, err) continue } addrIndex = append(addrIndex, toAppend...) } for _, txOut := range stx.MsgTx().TxOut { toAppend, err := convertToAddrIndex(txOut.Version, txOut.PkScript, blk.Height(), locInBlock) if err != nil { adxrLog.Tracef("Error converting stx txout %v: %v", stx.MsgTx().TxSha(), err) continue } addrIndex = append(addrIndex, toAppend...) } } return addrIndex, nil }
// InsertBlock inserts raw block and transaction data from a block into the // database. The first block inserted into the database will be treated as the // genesis block. Every subsequent block insert requires the referenced parent // block to already exist. func (db *LevelDb) InsertBlock(block *dcrutil.Block) (height int64, rerr error) { // Be careful with this function on syncs. It contains decred changes. // Obtain the previous block first so long as it's not the genesis block var blockPrev *dcrutil.Block // Decred: WARNING. This function assumes that all block insertion calls have // dcrutil.blocks passed to them with block.blockHeight set correctly. However, // loading the genesis block in btcd didn't do this (via block manager); pre- // production it should be established that all calls to this function pass // blocks with block.blockHeight set correctly. if block.Height() != 0 { var errBlockPrev error blockPrev, errBlockPrev = db.FetchBlockBySha(&block.MsgBlock().Header.PrevBlock) if errBlockPrev != nil { blockSha := block.Sha() log.Warnf("Failed to fetch parent block of block %v", blockSha) return 0, errBlockPrev } } db.dbLock.Lock() defer db.dbLock.Unlock() defer func() { if rerr == nil { rerr = db.processBatches() } else { db.lBatch().Reset() } }() blocksha := block.Sha() mblock := block.MsgBlock() rawMsg, err := block.Bytes() if err != nil { log.Warnf("Failed to obtain raw block sha %v", blocksha) return 0, err } _, sTxLoc, err := block.TxLoc() if err != nil { log.Warnf("Failed to obtain raw block sha %v, stxloc %v", blocksha, sTxLoc) return 0, err } // Insert block into database newheight, err := db.insertBlockData(blocksha, &mblock.Header.PrevBlock, rawMsg) if err != nil { log.Warnf("Failed to insert block %v %v %v", blocksha, &mblock.Header.PrevBlock, err) return 0, err } // Get data necessary to process regular tx tree of parent block if it's not // the genesis block. var mBlockPrev *wire.MsgBlock var txLoc []wire.TxLoc if blockPrev != nil { blockShaPrev := blockPrev.Sha() mBlockPrev = blockPrev.MsgBlock() txLoc, _, err = blockPrev.TxLoc() if err != nil { log.Warnf("Failed to obtain raw block sha %v, txloc %v", blockShaPrev, txLoc) return 0, err } } // Insert the regular tx of the parent block into the tx database if the vote // bits enable it, and if it's not the genesis block. votebits := mblock.Header.VoteBits if dcrutil.IsFlagSet16(votebits, dcrutil.BlockValid) && blockPrev != nil { for txidx, tx := range mBlockPrev.Transactions { txsha, err := blockPrev.TxSha(txidx) if err != nil { log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err) return 0, err } spentbuflen := (len(tx.TxOut) + 7) / 8 spentbuf := make([]byte, spentbuflen, spentbuflen) if len(tx.TxOut)%8 != 0 { for i := uint(len(tx.TxOut) % 8); i < 8; i++ { spentbuf[spentbuflen-1] |= (byte(1) << i) } } // newheight-1 instead of newheight below, as the tx is actually found // in the parent. //fmt.Printf("insert tx %v into db at height %v\n", txsha, newheight) err = db.insertTx(txsha, newheight-1, uint32(txidx), txLoc[txidx].TxStart, txLoc[txidx].TxLen, spentbuf) if err != nil { log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight-1, &txsha, txidx, err) return 0, err } err = db.doSpend(tx) if err != nil { log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, txsha, txidx, err) return 0, err } } } // Insert the stake tx of the current block into the tx database. if len(mblock.STransactions) != 0 { for txidx, tx := range mblock.STransactions { txsha, err := block.STxSha(txidx) if err != nil { log.Warnf("failed to compute stake tx name block %v idx %v err %v", blocksha, txidx, err) return 0, err } spentbuflen := (len(tx.TxOut) + 7) / 8 spentbuf := make([]byte, spentbuflen, spentbuflen) if len(tx.TxOut)%8 != 0 { for i := uint(len(tx.TxOut) % 8); i < 8; i++ { spentbuf[spentbuflen-1] |= (byte(1) << i) } } err = db.insertTx(txsha, newheight, uint32(txidx), sTxLoc[txidx].TxStart, sTxLoc[txidx].TxLen, spentbuf) if err != nil { log.Warnf("block %v idx %v failed to insert stake tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) return 0, err } err = db.doSpend(tx) if err != nil { log.Warnf("block %v idx %v failed to spend stx %v %v err %v", blocksha, newheight, txsha, txidx, err) return 0, err } } } return newheight, nil }
// dbAddTxIndexEntries uses an existing database transaction to add a // transaction index entry for every transaction in the passed block. func dbAddTxIndexEntries(dbTx database.Tx, block, parent *dcrutil.Block, blockID uint32) error { // The offset and length of the transactions within the serialized // block, for the regular transactions of the parent (if added) // and the stake transactions of the current block. regularTxTreeValid := dcrutil.IsFlagSet16(block.MsgBlock().Header.VoteBits, dcrutil.BlockValid) var parentRegularTxs []*dcrutil.Tx var parentTxLocs []wire.TxLoc var parentBlockID uint32 if regularTxTreeValid && block.Height() > 1 { var err error parentRegularTxs = parent.Transactions() parentTxLocs, _, err = parent.TxLoc() if err != nil { return err } parentSha := parent.Sha() parentBlockID, err = dbFetchBlockIDByHash(dbTx, *parentSha) if err != nil { return err } } _, blockStxLocs, err := block.TxLoc() if err != nil { return err } allTxs := append(parentRegularTxs, block.STransactions()...) allTxsLocs := append(parentTxLocs, blockStxLocs...) stakeTxStartIdx := len(parentRegularTxs) // As an optimization, allocate a single slice big enough to hold all // of the serialized transaction index entries for the block and // serialize them directly into the slice. Then, pass the appropriate // subslice to the database to be written. This approach significantly // cuts down on the number of required allocations. offset := 0 serializedValues := make([]byte, len(allTxs)*txEntrySize) blockIDToUse := parentBlockID for i, tx := range allTxs { // Switch to using the newest block ID for the stake transactions, // since these are not from the parent. if i == stakeTxStartIdx { blockIDToUse = blockID } putTxIndexEntry(serializedValues[offset:], blockIDToUse, allTxsLocs[i]) endOffset := offset + txEntrySize txSha := tx.Sha() err := dbPutTxIndexEntry(dbTx, *txSha, serializedValues[offset:endOffset:endOffset]) if err != nil { return err } offset += txEntrySize } return nil }
// testAddrIndexOperations ensures that all normal operations concerning // the optional address index function correctly. func testAddrIndexOperations(t *testing.T, db database.Db, newestBlock *dcrutil.Block, newestSha *chainhash.Hash, newestBlockIdx int64) { // Metadata about the current addr index state should be unset. sha, height, err := db.FetchAddrIndexTip() if err != database.ErrAddrIndexDoesNotExist { t.Fatalf("Address index metadata shouldn't be in db, hasn't been built up yet.") } var zeroHash chainhash.Hash if !sha.IsEqual(&zeroHash) { t.Fatalf("AddrIndexTip wrong hash got: %s, want %s", sha, &zeroHash) } if height != -1 { t.Fatalf("Addrindex not built up, yet a block index tip has been set to: %d.", height) } // Test enforcement of constraints for "limit" and "skip" var fakeAddr dcrutil.Address _, err = db.FetchTxsForAddr(fakeAddr, -1, 0) if err == nil { t.Fatalf("Negative value for skip passed, should return an error") } _, err = db.FetchTxsForAddr(fakeAddr, 0, -1) if err == nil { t.Fatalf("Negative value for limit passed, should return an error") } // Simple test to index outputs(s) of the first tx. testIndex := make(database.BlockAddrIndex, database.AddrIndexKeySize) testTx, err := newestBlock.Tx(0) if err != nil { t.Fatalf("Block has no transactions, unable to test addr "+ "indexing, err %v", err) } // Extract the dest addr from the tx. _, testAddrs, _, err := txscript.ExtractPkScriptAddrs(testTx.MsgTx().TxOut[0].Version, testTx.MsgTx().TxOut[0].PkScript, &chaincfg.MainNetParams) if err != nil { t.Fatalf("Unable to decode tx output, err %v", err) } // Extract the hash160 from the output script. var hash160Bytes [ripemd160.Size]byte testHash160 := testAddrs[0].(*dcrutil.AddressScriptHash).Hash160() copy(hash160Bytes[:], testHash160[:]) // Create a fake index. blktxLoc, _, _ := newestBlock.TxLoc() testIndex = []*database.TxAddrIndex{ &database.TxAddrIndex{ Hash160: hash160Bytes, Height: uint32(newestBlockIdx), TxOffset: uint32(blktxLoc[0].TxStart), TxLen: uint32(blktxLoc[0].TxLen), }, } // Insert our test addr index into the DB. err = db.UpdateAddrIndexForBlock(newestSha, newestBlockIdx, testIndex) if err != nil { t.Fatalf("UpdateAddrIndexForBlock: failed to index"+ " addrs for block #%d (%s) "+ "err %v", newestBlockIdx, newestSha, err) } // Chain Tip of address should've been updated. assertAddrIndexTipIsUpdated(db, t, newestSha, newestBlockIdx) // Check index retrieval. txReplies, err := db.FetchTxsForAddr(testAddrs[0], 0, 1000) if err != nil { t.Fatalf("FetchTxsForAddr failed to correctly fetch txs for an "+ "address, err %v", err) } // Should have one reply. if len(txReplies) != 1 { t.Fatalf("Failed to properly index tx by address.") } // Our test tx and indexed tx should have the same sha. indexedTx := txReplies[0] if !bytes.Equal(indexedTx.Sha.Bytes(), testTx.Sha().Bytes()) { t.Fatalf("Failed to fetch proper indexed tx. Expected sha %v, "+ "fetched %v", testTx.Sha(), indexedTx.Sha) } // Shut down DB. db.Sync() db.Close() // Re-Open, tip still should be updated to current height and sha. db, err = database.OpenDB("leveldb", "tstdbopmode") if err != nil { t.Fatalf("Unable to re-open created db, err %v", err) } assertAddrIndexTipIsUpdated(db, t, newestSha, newestBlockIdx) // Delete the entire index. err = db.PurgeAddrIndex() if err != nil { t.Fatalf("Couldn't delete address index, err %v", err) } // Former index should no longer exist. txReplies, err = db.FetchTxsForAddr(testAddrs[0], 0, 1000) if err != nil { t.Fatalf("Unable to fetch transactions for address: %v", err) } if len(txReplies) != 0 { t.Fatalf("Address index was not successfully deleted. "+ "Should have 0 tx's indexed, %v were returned.", len(txReplies)) } // Tip should be blanked out. if _, _, err := db.FetchAddrIndexTip(); err != database.ErrAddrIndexDoesNotExist { t.Fatalf("Address index was not fully deleted.") } }
// ConnectBlock is invoked by the index manager when a new block has been // connected to the main chain. This indexer adds a mapping for each address // the transactions in the block involve. // // This is part of the Indexer interface. func (idx *AddrIndex) ConnectBlock(dbTx database.Tx, block, parent *dcrutil.Block, view *blockchain.UtxoViewpoint) error { // The offset and length of the transactions within the serialized // block for the regular transactions of the previous block, if // applicable. regularTxTreeValid := dcrutil.IsFlagSet16(block.MsgBlock().Header.VoteBits, dcrutil.BlockValid) var parentTxLocs []wire.TxLoc var parentBlockID uint32 if regularTxTreeValid && block.Height() > 1 { var err error parentTxLocs, _, err = parent.TxLoc() if err != nil { return err } parentSha := parent.Sha() parentBlockID, err = dbFetchBlockIDByHash(dbTx, *parentSha) if err != nil { return err } } // The offset and length of the transactions within the serialized // block for the added stake transactions. _, blockStxLocs, err := block.TxLoc() if err != nil { return err } // Nothing to index, just return. if len(parentTxLocs)+len(blockStxLocs) == 0 { return nil } // Get the internal block ID associated with the block. blockSha := block.Sha() blockID, err := dbFetchBlockIDByHash(dbTx, *blockSha) if err != nil { return err } // Build all of the address to transaction mappings in a local map. addrsToTxns := make(writeIndexData) idx.indexBlock(addrsToTxns, block, parent, view) // Add all of the index entries for each address. stakeIdxsStart := len(parentTxLocs) allTxLocs := append(parentTxLocs, blockStxLocs...) addrIdxBucket := dbTx.Metadata().Bucket(addrIndexKey) for addrKey, txIdxs := range addrsToTxns { for _, txIdx := range txIdxs { // Switch to using the newest block ID for the stake transactions, // since these are not from the parent. Offset the index to be // correct for the location in this given block. blockIDToUse := parentBlockID if txIdx >= stakeIdxsStart { blockIDToUse = blockID } err := dbPutAddrIndexEntry(addrIdxBucket, addrKey, blockIDToUse, allTxLocs[txIdx]) if err != nil { return err } } } return nil }