// indexBlockAddrs returns a populated index of the all the transactions in the
// passed block based on the addresses involved in each transaction.
func (a *addrIndexer) indexBlockAddrs(blk *dcrutil.Block,
parent *dcrutil.Block) (database.BlockAddrIndex, error) {
var addrIndex database.BlockAddrIndex
_, stxLocs, err := blk.TxLoc()
if err != nil {
return nil, err
}
txTreeRegularValid := dcrutil.IsFlagSet16(blk.MsgBlock().Header.VoteBits,
dcrutil.BlockValid)
// Add regular transactions iff the block was validated.
if txTreeRegularValid {
txLocs, _, err := parent.TxLoc()
if err != nil {
return nil, err
}
for txIdx, tx := range parent.Transactions() {
// Tx's offset and length in the block.
locInBlock := &txLocs[txIdx]
// Coinbases don't have any inputs.
if !blockchain.IsCoinBase(tx) {
// Index the SPK's of each input's previous outpoint
// transaction.
for _, txIn := range tx.MsgTx().TxIn {
prevOutTx, err := a.lookupTransaction(
txIn.PreviousOutPoint.Hash,
blk,
parent)
inputOutPoint := prevOutTx.TxOut[txIn.PreviousOutPoint.Index]
toAppend, err := convertToAddrIndex(inputOutPoint.Version,
inputOutPoint.PkScript, parent.Height(), locInBlock)
if err != nil {
adxrLog.Tracef("Error converting tx txin %v: %v",
txIn.PreviousOutPoint.Hash, err)
continue
}
addrIndex = append(addrIndex, toAppend...)
}
}
for _, txOut := range tx.MsgTx().TxOut {
toAppend, err := convertToAddrIndex(txOut.Version, txOut.PkScript,
parent.Height(), locInBlock)
if err != nil {
adxrLog.Tracef("Error converting tx txout %v: %v",
tx.MsgTx().TxSha(), err)
continue
}
addrIndex = append(addrIndex, toAppend...)
}
}
}
// Add stake transactions.
for stxIdx, stx := range blk.STransactions() {
// Tx's offset and length in the block.
locInBlock := &stxLocs[stxIdx]
isSSGen, _ := stake.IsSSGen(stx)
// Index the SPK's of each input's previous outpoint
// transaction.
for i, txIn := range stx.MsgTx().TxIn {
// Stakebases don't have any inputs.
if isSSGen && i == 0 {
continue
}
// Lookup and fetch the referenced output's tx.
prevOutTx, err := a.lookupTransaction(
txIn.PreviousOutPoint.Hash,
blk,
parent)
inputOutPoint := prevOutTx.TxOut[txIn.PreviousOutPoint.Index]
toAppend, err := convertToAddrIndex(inputOutPoint.Version,
inputOutPoint.PkScript, blk.Height(), locInBlock)
if err != nil {
adxrLog.Tracef("Error converting stx txin %v: %v",
txIn.PreviousOutPoint.Hash, err)
continue
}
addrIndex = append(addrIndex, toAppend...)
}
for _, txOut := range stx.MsgTx().TxOut {
toAppend, err := convertToAddrIndex(txOut.Version, txOut.PkScript,
blk.Height(), locInBlock)
if err != nil {
adxrLog.Tracef("Error converting stx txout %v: %v",
stx.MsgTx().TxSha(), err)
continue
}
addrIndex = append(addrIndex, toAppend...)
}
}
return addrIndex, nil
}
// InsertBlock inserts raw block and transaction data from a block into the
// database. The first block inserted into the database will be treated as the
// genesis block. Every subsequent block insert requires the referenced parent
// block to already exist.
func (db *LevelDb) InsertBlock(block *dcrutil.Block) (height int64, rerr error) {
// Be careful with this function on syncs. It contains decred changes.
// Obtain the previous block first so long as it's not the genesis block
var blockPrev *dcrutil.Block
// Decred: WARNING. This function assumes that all block insertion calls have
// dcrutil.blocks passed to them with block.blockHeight set correctly. However,
// loading the genesis block in btcd didn't do this (via block manager); pre-
// production it should be established that all calls to this function pass
// blocks with block.blockHeight set correctly.
if block.Height() != 0 {
var errBlockPrev error
blockPrev, errBlockPrev = db.FetchBlockBySha(&block.MsgBlock().Header.PrevBlock)
if errBlockPrev != nil {
blockSha := block.Sha()
log.Warnf("Failed to fetch parent block of block %v", blockSha)
return 0, errBlockPrev
}
}
db.dbLock.Lock()
defer db.dbLock.Unlock()
defer func() {
if rerr == nil {
rerr = db.processBatches()
} else {
db.lBatch().Reset()
}
}()
blocksha := block.Sha()
mblock := block.MsgBlock()
rawMsg, err := block.Bytes()
if err != nil {
log.Warnf("Failed to obtain raw block sha %v", blocksha)
return 0, err
}
_, sTxLoc, err := block.TxLoc()
if err != nil {
log.Warnf("Failed to obtain raw block sha %v, stxloc %v", blocksha, sTxLoc)
return 0, err
}
// Insert block into database
newheight, err := db.insertBlockData(blocksha, &mblock.Header.PrevBlock,
rawMsg)
if err != nil {
log.Warnf("Failed to insert block %v %v %v", blocksha,
&mblock.Header.PrevBlock, err)
return 0, err
}
// Get data necessary to process regular tx tree of parent block if it's not
// the genesis block.
var mBlockPrev *wire.MsgBlock
var txLoc []wire.TxLoc
if blockPrev != nil {
blockShaPrev := blockPrev.Sha()
mBlockPrev = blockPrev.MsgBlock()
txLoc, _, err = blockPrev.TxLoc()
if err != nil {
log.Warnf("Failed to obtain raw block sha %v, txloc %v", blockShaPrev, txLoc)
return 0, err
}
}
// Insert the regular tx of the parent block into the tx database if the vote
// bits enable it, and if it's not the genesis block.
votebits := mblock.Header.VoteBits
if dcrutil.IsFlagSet16(votebits, dcrutil.BlockValid) && blockPrev != nil {
for txidx, tx := range mBlockPrev.Transactions {
txsha, err := blockPrev.TxSha(txidx)
if err != nil {
log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err)
return 0, err
}
spentbuflen := (len(tx.TxOut) + 7) / 8
spentbuf := make([]byte, spentbuflen, spentbuflen)
if len(tx.TxOut)%8 != 0 {
for i := uint(len(tx.TxOut) % 8); i < 8; i++ {
spentbuf[spentbuflen-1] |= (byte(1) << i)
}
}
// newheight-1 instead of newheight below, as the tx is actually found
// in the parent.
//fmt.Printf("insert tx %v into db at height %v\n", txsha, newheight)
err = db.insertTx(txsha, newheight-1, uint32(txidx), txLoc[txidx].TxStart, txLoc[txidx].TxLen, spentbuf)
if err != nil {
log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight-1, &txsha, txidx, err)
return 0, err
}
err = db.doSpend(tx)
if err != nil {
log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, txsha, txidx, err)
return 0, err
}
}
}
// Insert the stake tx of the current block into the tx database.
if len(mblock.STransactions) != 0 {
for txidx, tx := range mblock.STransactions {
txsha, err := block.STxSha(txidx)
if err != nil {
log.Warnf("failed to compute stake tx name block %v idx %v err %v", blocksha, txidx, err)
return 0, err
}
spentbuflen := (len(tx.TxOut) + 7) / 8
spentbuf := make([]byte, spentbuflen, spentbuflen)
if len(tx.TxOut)%8 != 0 {
for i := uint(len(tx.TxOut) % 8); i < 8; i++ {
spentbuf[spentbuflen-1] |= (byte(1) << i)
}
}
err = db.insertTx(txsha, newheight, uint32(txidx), sTxLoc[txidx].TxStart, sTxLoc[txidx].TxLen, spentbuf)
if err != nil {
log.Warnf("block %v idx %v failed to insert stake tx %v %v err %v", blocksha, newheight, &txsha, txidx, err)
return 0, err
}
err = db.doSpend(tx)
if err != nil {
log.Warnf("block %v idx %v failed to spend stx %v %v err %v", blocksha, newheight, txsha, txidx, err)
return 0, err
}
}
}
return newheight, nil
}
// dbAddTxIndexEntries uses an existing database transaction to add a
// transaction index entry for every transaction in the passed block.
func dbAddTxIndexEntries(dbTx database.Tx, block, parent *dcrutil.Block, blockID uint32) error {
// The offset and length of the transactions within the serialized
// block, for the regular transactions of the parent (if added)
// and the stake transactions of the current block.
regularTxTreeValid := dcrutil.IsFlagSet16(block.MsgBlock().Header.VoteBits,
dcrutil.BlockValid)
var parentRegularTxs []*dcrutil.Tx
var parentTxLocs []wire.TxLoc
var parentBlockID uint32
if regularTxTreeValid && block.Height() > 1 {
var err error
parentRegularTxs = parent.Transactions()
parentTxLocs, _, err = parent.TxLoc()
if err != nil {
return err
}
parentSha := parent.Sha()
parentBlockID, err = dbFetchBlockIDByHash(dbTx, *parentSha)
if err != nil {
return err
}
}
_, blockStxLocs, err := block.TxLoc()
if err != nil {
return err
}
allTxs := append(parentRegularTxs, block.STransactions()...)
allTxsLocs := append(parentTxLocs, blockStxLocs...)
stakeTxStartIdx := len(parentRegularTxs)
// As an optimization, allocate a single slice big enough to hold all
// of the serialized transaction index entries for the block and
// serialize them directly into the slice. Then, pass the appropriate
// subslice to the database to be written. This approach significantly
// cuts down on the number of required allocations.
offset := 0
serializedValues := make([]byte, len(allTxs)*txEntrySize)
blockIDToUse := parentBlockID
for i, tx := range allTxs {
// Switch to using the newest block ID for the stake transactions,
// since these are not from the parent.
if i == stakeTxStartIdx {
blockIDToUse = blockID
}
putTxIndexEntry(serializedValues[offset:], blockIDToUse, allTxsLocs[i])
endOffset := offset + txEntrySize
txSha := tx.Sha()
err := dbPutTxIndexEntry(dbTx, *txSha,
serializedValues[offset:endOffset:endOffset])
if err != nil {
return err
}
offset += txEntrySize
}
return nil
}
// testAddrIndexOperations ensures that all normal operations concerning
// the optional address index function correctly.
func testAddrIndexOperations(t *testing.T, db database.Db, newestBlock *dcrutil.Block, newestSha *chainhash.Hash, newestBlockIdx int64) {
// Metadata about the current addr index state should be unset.
sha, height, err := db.FetchAddrIndexTip()
if err != database.ErrAddrIndexDoesNotExist {
t.Fatalf("Address index metadata shouldn't be in db, hasn't been built up yet.")
}
var zeroHash chainhash.Hash
if !sha.IsEqual(&zeroHash) {
t.Fatalf("AddrIndexTip wrong hash got: %s, want %s", sha, &zeroHash)
}
if height != -1 {
t.Fatalf("Addrindex not built up, yet a block index tip has been set to: %d.", height)
}
// Test enforcement of constraints for "limit" and "skip"
var fakeAddr dcrutil.Address
_, err = db.FetchTxsForAddr(fakeAddr, -1, 0)
if err == nil {
t.Fatalf("Negative value for skip passed, should return an error")
}
_, err = db.FetchTxsForAddr(fakeAddr, 0, -1)
if err == nil {
t.Fatalf("Negative value for limit passed, should return an error")
}
// Simple test to index outputs(s) of the first tx.
testIndex := make(database.BlockAddrIndex, database.AddrIndexKeySize)
testTx, err := newestBlock.Tx(0)
if err != nil {
t.Fatalf("Block has no transactions, unable to test addr "+
"indexing, err %v", err)
}
// Extract the dest addr from the tx.
_, testAddrs, _, err := txscript.ExtractPkScriptAddrs(testTx.MsgTx().TxOut[0].Version, testTx.MsgTx().TxOut[0].PkScript, &chaincfg.MainNetParams)
if err != nil {
t.Fatalf("Unable to decode tx output, err %v", err)
}
// Extract the hash160 from the output script.
var hash160Bytes [ripemd160.Size]byte
testHash160 := testAddrs[0].(*dcrutil.AddressScriptHash).Hash160()
copy(hash160Bytes[:], testHash160[:])
// Create a fake index.
blktxLoc, _, _ := newestBlock.TxLoc()
testIndex = []*database.TxAddrIndex{
&database.TxAddrIndex{
Hash160: hash160Bytes,
Height: uint32(newestBlockIdx),
TxOffset: uint32(blktxLoc[0].TxStart),
TxLen: uint32(blktxLoc[0].TxLen),
},
}
// Insert our test addr index into the DB.
err = db.UpdateAddrIndexForBlock(newestSha, newestBlockIdx, testIndex)
if err != nil {
t.Fatalf("UpdateAddrIndexForBlock: failed to index"+
" addrs for block #%d (%s) "+
"err %v", newestBlockIdx, newestSha, err)
}
// Chain Tip of address should've been updated.
assertAddrIndexTipIsUpdated(db, t, newestSha, newestBlockIdx)
// Check index retrieval.
txReplies, err := db.FetchTxsForAddr(testAddrs[0], 0, 1000)
if err != nil {
t.Fatalf("FetchTxsForAddr failed to correctly fetch txs for an "+
"address, err %v", err)
}
// Should have one reply.
if len(txReplies) != 1 {
t.Fatalf("Failed to properly index tx by address.")
}
// Our test tx and indexed tx should have the same sha.
indexedTx := txReplies[0]
if !bytes.Equal(indexedTx.Sha.Bytes(), testTx.Sha().Bytes()) {
t.Fatalf("Failed to fetch proper indexed tx. Expected sha %v, "+
"fetched %v", testTx.Sha(), indexedTx.Sha)
}
// Shut down DB.
db.Sync()
db.Close()
// Re-Open, tip still should be updated to current height and sha.
db, err = database.OpenDB("leveldb", "tstdbopmode")
if err != nil {
t.Fatalf("Unable to re-open created db, err %v", err)
}
assertAddrIndexTipIsUpdated(db, t, newestSha, newestBlockIdx)
// Delete the entire index.
err = db.PurgeAddrIndex()
if err != nil {
t.Fatalf("Couldn't delete address index, err %v", err)
}
// Former index should no longer exist.
txReplies, err = db.FetchTxsForAddr(testAddrs[0], 0, 1000)
if err != nil {
t.Fatalf("Unable to fetch transactions for address: %v", err)
}
if len(txReplies) != 0 {
t.Fatalf("Address index was not successfully deleted. "+
"Should have 0 tx's indexed, %v were returned.",
len(txReplies))
}
// Tip should be blanked out.
if _, _, err := db.FetchAddrIndexTip(); err != database.ErrAddrIndexDoesNotExist {
t.Fatalf("Address index was not fully deleted.")
}
}
// ConnectBlock is invoked by the index manager when a new block has been
// connected to the main chain. This indexer adds a mapping for each address
// the transactions in the block involve.
//
// This is part of the Indexer interface.
func (idx *AddrIndex) ConnectBlock(dbTx database.Tx, block, parent *dcrutil.Block,
view *blockchain.UtxoViewpoint) error {
// The offset and length of the transactions within the serialized
// block for the regular transactions of the previous block, if
// applicable.
regularTxTreeValid := dcrutil.IsFlagSet16(block.MsgBlock().Header.VoteBits,
dcrutil.BlockValid)
var parentTxLocs []wire.TxLoc
var parentBlockID uint32
if regularTxTreeValid && block.Height() > 1 {
var err error
parentTxLocs, _, err = parent.TxLoc()
if err != nil {
return err
}
parentSha := parent.Sha()
parentBlockID, err = dbFetchBlockIDByHash(dbTx, *parentSha)
if err != nil {
return err
}
}
// The offset and length of the transactions within the serialized
// block for the added stake transactions.
_, blockStxLocs, err := block.TxLoc()
if err != nil {
return err
}
// Nothing to index, just return.
if len(parentTxLocs)+len(blockStxLocs) == 0 {
return nil
}
// Get the internal block ID associated with the block.
blockSha := block.Sha()
blockID, err := dbFetchBlockIDByHash(dbTx, *blockSha)
if err != nil {
return err
}
// Build all of the address to transaction mappings in a local map.
addrsToTxns := make(writeIndexData)
idx.indexBlock(addrsToTxns, block, parent, view)
// Add all of the index entries for each address.
stakeIdxsStart := len(parentTxLocs)
allTxLocs := append(parentTxLocs, blockStxLocs...)
addrIdxBucket := dbTx.Metadata().Bucket(addrIndexKey)
for addrKey, txIdxs := range addrsToTxns {
for _, txIdx := range txIdxs {
// Switch to using the newest block ID for the stake transactions,
// since these are not from the parent. Offset the index to be
// correct for the location in this given block.
blockIDToUse := parentBlockID
if txIdx >= stakeIdxsStart {
blockIDToUse = blockID
}
err := dbPutAddrIndexEntry(addrIdxBucket, addrKey,
blockIDToUse, allTxLocs[txIdx])
if err != nil {
return err
}
}
}
return nil
}