// maybeCreateIndexes determines if each of the enabled indexes have already
// been created and creates them if not.
func (m *Manager) maybeCreateIndexes(dbTx database.Tx) error {
indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
for _, indexer := range m.enabledIndexes {
// Nothing to do if the index tip already exists.
idxKey := indexer.Key()
if indexesBucket.Get(idxKey) != nil {
continue
}
// The tip for the index does not exist, so create it and
// invoke the create callback for the index so it can perform
// any one-time initialization it requires.
if err := indexer.Create(dbTx); err != nil {
return err
}
// Set the tip for the index to values which represent an
// uninitialized index (the genesis block hack and height).
genesisBlockHash := m.params.GenesisBlock.BlockSha()
err := dbPutIndexerTip(dbTx, idxKey, &genesisBlockHash, 0)
if err != nil {
return err
}
}
return nil
}
// DbPutBestState uses an existing database transaction to update the best chain
// state with the given parameters.
func DbPutBestState(dbTx database.Tx, bcs BestChainState) error {
// Serialize the current best chain state.
serializedData := serializeBestChainState(bcs)
// Store the current best chain state into the database.
return dbTx.Metadata().Put(dbnamespace.StakeChainStateKeyName, serializedData)
}
// TxRegionsForAddress returns a slice of block regions which identify each
// transaction that involves the passed address according to the specified
// number to skip, number requested, and whether or not the results should be
// reversed. It also returns the number actually skipped since it could be less
// in the case where there are not enough entries.
//
// NOTE: These results only include transactions confirmed in blocks. See the
// UnconfirmedTxnsForAddress method for obtaining unconfirmed transactions
// that involve a given address.
//
// This function is safe for concurrent access.
func (idx *AddrIndex) TxRegionsForAddress(dbTx database.Tx, addr dcrutil.Address, numToSkip, numRequested uint32, reverse bool) ([]database.BlockRegion, uint32, error) {
addrKey, err := addrToKey(addr, idx.chainParams)
if err != nil {
return nil, 0, err
}
var regions []database.BlockRegion
var skipped uint32
err = idx.db.View(func(dbTx database.Tx) error {
// Create closure to lookup the block hash given the ID using
// the database transaction.
fetchBlockHash := func(id []byte) (chainhash.Hash, error) {
// Deserialize and populate the result.
return dbFetchBlockHashBySerializedID(dbTx, id)
}
var err error
addrIdxBucket := dbTx.Metadata().Bucket(addrIndexKey)
regions, skipped, err = dbFetchAddrIndexEntries(addrIdxBucket,
addrKey, numToSkip, numRequested, reverse,
fetchBlockHash)
return err
})
return regions, skipped, err
}
// dbFetchTxIndexEntry uses an existing database transaction to fetch the block
// region for the provided transaction hash from the transaction index. When
// there is no entry for the provided hash, nil will be returned for the both
// the region and the error.
func dbFetchTxIndexEntry(dbTx database.Tx, txHash chainhash.Hash) (*database.BlockRegion, error) {
// Load the record from the database and return now if it doesn't exist.
txIndex := dbTx.Metadata().Bucket(txIndexKey)
serializedData := txIndex.Get(txHash[:])
if len(serializedData) == 0 {
return nil, nil
}
// Ensure the serialized data has enough bytes to properly deserialize.
if len(serializedData) < 12 {
return nil, database.Error{
ErrorCode: database.ErrCorruption,
Description: fmt.Sprintf("corrupt transaction index "+
"entry for %s", txHash),
}
}
// Load the block hash associated with the block ID.
hash, err := dbFetchBlockHashBySerializedID(dbTx, serializedData[0:4])
if err != nil {
return nil, database.Error{
ErrorCode: database.ErrCorruption,
Description: fmt.Sprintf("corrupt transaction index "+
"entry for %s: %v", txHash, err),
}
}
// Deserialize the final entry.
region := database.BlockRegion{Hash: &chainhash.Hash{}}
copy(region.Hash[:], hash[:])
region.Offset = byteOrder.Uint32(serializedData[4:8])
region.Len = byteOrder.Uint32(serializedData[8:12])
return ®ion, nil
}
// DbLoadAllTickets loads all the live tickets from the database into a treap.
func DbLoadAllTickets(dbTx database.Tx, ticketBucket []byte) (*tickettreap.Immutable, error) {
meta := dbTx.Metadata()
bucket := meta.Bucket(ticketBucket)
treap := tickettreap.NewImmutable()
err := bucket.ForEach(func(k []byte, v []byte) error {
if len(v) < 5 {
return ticketDBError(ErrLoadAllTickets, fmt.Sprintf("short "+
"read for ticket key %x when loading tickets", k))
}
h, err := chainhash.NewHash(k)
if err != nil {
return err
}
treapKey := tickettreap.Key(*h)
missed, revoked, spent, expired := undoBitFlagsFromByte(v[4])
treapValue := &tickettreap.Value{
Height: dbnamespace.ByteOrder.Uint32(v[0:4]),
Missed: missed,
Revoked: revoked,
Spent: spent,
Expired: expired,
}
treap = treap.Put(treapKey, treapValue)
return nil
})
if err != nil {
return nil, ticketDBError(ErrLoadAllTickets, fmt.Sprintf("failed to "+
"load all tickets for the bucket %s", string(ticketBucket)))
}
return treap, nil
}
// DbPutDatabaseInfo uses an existing database transaction to store the database
// information.
func DbPutDatabaseInfo(dbTx database.Tx, dbi *DatabaseInfo) error {
meta := dbTx.Metadata()
subsidyBucket := meta.Bucket(dbnamespace.StakeDbInfoBucketName)
val := serializeDatabaseInfo(dbi)
// Store the current database info into the database.
return subsidyBucket.Put(dbnamespace.StakeDbInfoBucketName, val[:])
}
// dbPutIndexerTip uses an existing database transaction to update or add the
// current tip for the given index to the provided values.
func dbPutIndexerTip(dbTx database.Tx, idxKey []byte, hash *chainhash.Hash, height uint32) error {
serialized := make([]byte, chainhash.HashSize+4)
copy(serialized, hash[:])
byteOrder.PutUint32(serialized[chainhash.HashSize:], height)
indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
return indexesBucket.Put(idxKey, serialized)
}
// DbDropNewTickets drops new tickets for a mainchain block data at some height.
func DbDropNewTickets(dbTx database.Tx, height uint32) error {
meta := dbTx.Metadata()
bucket := meta.Bucket(dbnamespace.TicketsInBlockBucketName)
k := make([]byte, 4)
dbnamespace.ByteOrder.PutUint32(k, height)
return bucket.Delete(k)
}
// dbFetchBlockIDByHash uses an existing database transaction to retrieve the
// block id for the provided hash from the index.
func dbFetchBlockIDByHash(dbTx database.Tx, hash chainhash.Hash) (uint32, error) {
hashIndex := dbTx.Metadata().Bucket(idByHashIndexBucketName)
serializedID := hashIndex.Get(hash[:])
if serializedID == nil {
return 0, errNoBlockIDEntry
}
return byteOrder.Uint32(serializedID), nil
}
// DbPutBlockUndoData inserts block undo data into the database for a given height.
func DbPutBlockUndoData(dbTx database.Tx, height uint32, utds []UndoTicketData) error {
meta := dbTx.Metadata()
bucket := meta.Bucket(dbnamespace.StakeBlockUndoDataBucketName)
k := make([]byte, 4)
dbnamespace.ByteOrder.PutUint32(k, height)
v := serializeBlockUndoData(utds)
return bucket.Put(k[:], v[:])
}
// DbPutNewTickets inserts new tickets for a mainchain block data into the
// database.
func DbPutNewTickets(dbTx database.Tx, height uint32, ths TicketHashes) error {
meta := dbTx.Metadata()
bucket := meta.Bucket(dbnamespace.TicketsInBlockBucketName)
k := make([]byte, 4)
dbnamespace.ByteOrder.PutUint32(k, height)
v := serializeTicketHashes(ths)
return bucket.Put(k[:], v[:])
}
// DbFetchBestState uses an existing database transaction to fetch the best chain
// state.
func DbFetchBestState(dbTx database.Tx) (BestChainState, error) {
meta := dbTx.Metadata()
v := meta.Get(dbnamespace.StakeChainStateKeyName)
if v == nil {
return BestChainState{}, ticketDBError(ErrMissingKey,
"missing key for chain state data")
}
return deserializeBestChainState(v)
}
// dbRemoveTxIndexEntry uses an existing database transaction to remove the most
// recent transaction index entry for the given hash.
func dbRemoveTxIndexEntry(dbTx database.Tx, txHash chainhash.Hash) error {
txIndex := dbTx.Metadata().Bucket(txIndexKey)
serializedData := txIndex.Get(txHash[:])
if len(serializedData) == 0 {
return fmt.Errorf("can't remove non-existent transaction %s "+
"from the transaction index", txHash)
}
return txIndex.Delete(txHash[:])
}
// dbFetchBlockHashBySerializedID uses an existing database transaction to
// retrieve the hash for the provided serialized block id from the index.
func dbFetchBlockHashBySerializedID(dbTx database.Tx, serializedID []byte) (chainhash.Hash, error) {
idIndex := dbTx.Metadata().Bucket(hashByIDIndexBucketName)
hashBytes := idIndex.Get(serializedID)
if hashBytes == nil {
return chainhash.Hash{}, errNoBlockIDEntry
}
var hash chainhash.Hash
copy(hash[:], hashBytes)
return hash, nil
}
// DbCreate initializes all the buckets required for the database and stores
// the current database version information.
func DbCreate(dbTx database.Tx) error {
meta := dbTx.Metadata()
// Create the bucket that houses information about the database's
// creation and version.
_, err := meta.CreateBucket(dbnamespace.StakeDbInfoBucketName)
if err != nil {
return err
}
dbInfo := &DatabaseInfo{
Version: currentDatabaseVersion,
Date: time.Now(),
UpgradeStarted: false,
}
err = DbPutDatabaseInfo(dbTx, dbInfo)
if err != nil {
return err
}
// Create the bucket that houses the live tickets of the best node.
_, err = meta.CreateBucket(dbnamespace.LiveTicketsBucketName)
if err != nil {
return err
}
// Create the bucket that houses the missed tickets of the best node.
_, err = meta.CreateBucket(dbnamespace.MissedTicketsBucketName)
if err != nil {
return err
}
// Create the bucket that houses the revoked tickets of the best node.
_, err = meta.CreateBucket(dbnamespace.RevokedTicketsBucketName)
if err != nil {
return err
}
// Create the bucket that houses block undo data for stake states on
// the main chain.
_, err = meta.CreateBucket(dbnamespace.StakeBlockUndoDataBucketName)
if err != nil {
return err
}
// Create the bucket that houses the tickets that were added with
// this block into the main chain.
_, err = meta.CreateBucket(dbnamespace.TicketsInBlockBucketName)
if err != nil {
return err
}
return nil
}
// DbPutTicket inserts a ticket into one of the ticket database buckets.
func DbPutTicket(dbTx database.Tx, ticketBucket []byte, hash *chainhash.Hash,
height uint32, missed, revoked, spent, expired bool) error {
meta := dbTx.Metadata()
bucket := meta.Bucket(ticketBucket)
k := hash[:]
v := make([]byte, 5)
dbnamespace.ByteOrder.PutUint32(v, height)
v[4] = undoBitFlagsToByte(missed, revoked, spent, expired)
return bucket.Put(k[:], v[:])
}
// Create is invoked when the indexer manager determines the index needs
// to be created for the first time. It creates the buckets for the hash-based
// transaction index and the internal block ID indexes.
//
// This is part of the Indexer interface.
func (idx *TxIndex) Create(dbTx database.Tx) error {
meta := dbTx.Metadata()
if _, err := meta.CreateBucket(idByHashIndexBucketName); err != nil {
return err
}
if _, err := meta.CreateBucket(hashByIDIndexBucketName); err != nil {
return err
}
_, err := meta.CreateBucket(txIndexKey)
return err
}
// DbDeleteTicket removes a ticket from one of the ticket database buckets. This
// differs from the bucket deletion method in that it will fail if the value
// itself is missing.
func DbDeleteTicket(dbTx database.Tx, ticketBucket []byte, hash *chainhash.Hash) error {
meta := dbTx.Metadata()
bucket := meta.Bucket(ticketBucket)
// Check to see if the value exists before we delete it.
v := bucket.Get(hash[:])
if v == nil {
return ticketDBError(ErrMissingKey, fmt.Sprintf("missing key %v "+
"to delete", hash))
}
return bucket.Delete(hash[:])
}
// DbFetchBlockUndoData fetches block undo data from the database.
func DbFetchBlockUndoData(dbTx database.Tx, height uint32) ([]UndoTicketData, error) {
meta := dbTx.Metadata()
bucket := meta.Bucket(dbnamespace.StakeBlockUndoDataBucketName)
k := make([]byte, 4)
dbnamespace.ByteOrder.PutUint32(k, height)
v := bucket.Get(k)
if v == nil {
return nil, ticketDBError(ErrMissingKey,
fmt.Sprintf("missing key %v for block undo data", height))
}
return deserializeBlockUndoData(v)
}
// DbFetchNewTickets fetches new tickets for a mainchain block from the database.
func DbFetchNewTickets(dbTx database.Tx, height uint32) (TicketHashes, error) {
meta := dbTx.Metadata()
bucket := meta.Bucket(dbnamespace.TicketsInBlockBucketName)
k := make([]byte, 4)
dbnamespace.ByteOrder.PutUint32(k, height)
v := bucket.Get(k)
if v == nil {
return nil, ticketDBError(ErrMissingKey,
fmt.Sprintf("missing key %v for new tickets", height))
}
return deserializeTicketHashes(v)
}
// dbPutBlockIDIndexEntry uses an existing database transaction to update or add
// the index entries for the hash to id and id to hash mappings for the provided
// values.
func dbPutBlockIDIndexEntry(dbTx database.Tx, hash chainhash.Hash, id uint32) error {
// Serialize the height for use in the index entries.
var serializedID [4]byte
byteOrder.PutUint32(serializedID[:], id)
// Add the block hash to ID mapping to the index.
meta := dbTx.Metadata()
hashIndex := meta.Bucket(idByHashIndexBucketName)
if err := hashIndex.Put(hash[:], serializedID[:]); err != nil {
return err
}
// Add the block ID to hash mapping to the index.
idIndex := meta.Bucket(hashByIDIndexBucketName)
return idIndex.Put(serializedID[:], hash[:])
}
// DbFetchDatabaseInfo uses an existing database transaction to
// fetch the database versioning and creation information.
func DbFetchDatabaseInfo(dbTx database.Tx) (*DatabaseInfo, error) {
meta := dbTx.Metadata()
bucket := meta.Bucket(dbnamespace.StakeDbInfoBucketName)
// Uninitialized state.
if bucket == nil {
return nil, nil
}
dbInfoBytes := bucket.Get(dbnamespace.StakeDbInfoBucketName)
if dbInfoBytes == nil {
return nil, ticketDBError(ErrMissingKey, "missing key for database info")
}
return deserializeDatabaseInfo(dbInfoBytes)
}
// dbRemoveBlockIDIndexEntry uses an existing database transaction remove index
// entries from the hash to id and id to hash mappings for the provided hash.
func dbRemoveBlockIDIndexEntry(dbTx database.Tx, hash chainhash.Hash) error {
// Remove the block hash to ID mapping.
meta := dbTx.Metadata()
hashIndex := meta.Bucket(idByHashIndexBucketName)
serializedID := hashIndex.Get(hash[:])
if serializedID == nil {
return nil
}
if err := hashIndex.Delete(hash[:]); err != nil {
return err
}
// Remove the block ID to hash mapping.
idIndex := meta.Bucket(hashByIDIndexBucketName)
return idIndex.Delete(serializedID)
}
// dbFetchIndexerTip uses an existing database transaction to retrieve the
// hash and height of the current tip for the provided index.
func dbFetchIndexerTip(dbTx database.Tx, idxKey []byte) (*chainhash.Hash, uint32, error) {
indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
serialized := indexesBucket.Get(idxKey)
if len(serialized) < chainhash.HashSize+4 {
return nil, 0, database.Error{
ErrorCode: database.ErrCorruption,
Description: fmt.Sprintf("unexpected end of data for "+
"index %q tip", string(idxKey)),
}
}
var hash chainhash.Hash
copy(hash[:], serialized[:chainhash.HashSize])
height := byteOrder.Uint32(serialized[chainhash.HashSize:])
return &hash, height, nil
}
// DisconnectBlock is invoked by the index manager when a block has been
// disconnected from the main chain. This indexer removes the address mappings
// each transaction in the block involve.
//
// This is part of the Indexer interface.
func (idx *AddrIndex) DisconnectBlock(dbTx database.Tx, block, parent *dcrutil.Block, view *blockchain.UtxoViewpoint) error {
// Build all of the address to transaction mappings in a local map.
addrsToTxns := make(writeIndexData)
idx.indexBlock(addrsToTxns, block, parent, view)
// Remove all of the index entries for each address.
bucket := dbTx.Metadata().Bucket(addrIndexKey)
for addrKey, txIdxs := range addrsToTxns {
err := dbRemoveAddrIndexEntries(bucket, addrKey, len(txIdxs))
if err != nil {
return err
}
}
return nil
}
// ConnectBlock is invoked by the index manager when a new block has been
// connected to the main chain. This indexer adds a mapping for each address
// the transactions in the block involve.
//
// This is part of the Indexer interface.
func (idx *AddrIndex) ConnectBlock(dbTx database.Tx, block, parent *dcrutil.Block,
view *blockchain.UtxoViewpoint) error {
// The offset and length of the transactions within the serialized
// block for the regular transactions of the previous block, if
// applicable.
regularTxTreeValid := dcrutil.IsFlagSet16(block.MsgBlock().Header.VoteBits,
dcrutil.BlockValid)
var parentTxLocs []wire.TxLoc
var parentBlockID uint32
if regularTxTreeValid && block.Height() > 1 {
var err error
parentTxLocs, _, err = parent.TxLoc()
if err != nil {
return err
}
parentSha := parent.Sha()
parentBlockID, err = dbFetchBlockIDByHash(dbTx, *parentSha)
if err != nil {
return err
}
}
// The offset and length of the transactions within the serialized
// block for the added stake transactions.
_, blockStxLocs, err := block.TxLoc()
if err != nil {
return err
}
// Nothing to index, just return.
if len(parentTxLocs)+len(blockStxLocs) == 0 {
return nil
}
// Get the internal block ID associated with the block.
blockSha := block.Sha()
blockID, err := dbFetchBlockIDByHash(dbTx, *blockSha)
if err != nil {
return err
}
// Build all of the address to transaction mappings in a local map.
addrsToTxns := make(writeIndexData)
idx.indexBlock(addrsToTxns, block, parent, view)
// Add all of the index entries for each address.
stakeIdxsStart := len(parentTxLocs)
allTxLocs := append(parentTxLocs, blockStxLocs...)
addrIdxBucket := dbTx.Metadata().Bucket(addrIndexKey)
for addrKey, txIdxs := range addrsToTxns {
for _, txIdx := range txIdxs {
// Switch to using the newest block ID for the stake transactions,
// since these are not from the parent. Offset the index to be
// correct for the location in this given block.
blockIDToUse := parentBlockID
if txIdx >= stakeIdxsStart {
blockIDToUse = blockID
}
err := dbPutAddrIndexEntry(addrIdxBucket, addrKey,
blockIDToUse, allTxLocs[txIdx])
if err != nil {
return err
}
}
}
return nil
}
// ConnectBlock is invoked by the index manager when a new block has been
// connected to the main chain. This indexer adds a key for each address
// the transactions in the block involve.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) ConnectBlock(dbTx database.Tx, block, parent *dcrutil.Block, view *blockchain.UtxoViewpoint) error {
regularTxTreeValid := dcrutil.IsFlagSet16(block.MsgBlock().Header.VoteBits,
dcrutil.BlockValid)
var parentTxs []*dcrutil.Tx
if regularTxTreeValid && block.Height() > 1 {
parentTxs = parent.Transactions()
}
blockTxns := block.STransactions()
allTxns := append(parentTxs, blockTxns...)
usedAddrs := make(map[[addrKeySize]byte]struct{})
for _, tx := range allTxns {
msgTx := tx.MsgTx()
isSStx, _ := stake.IsSStx(msgTx)
for _, txIn := range msgTx.TxIn {
if txscript.IsMultisigSigScript(txIn.SignatureScript) {
rs, err :=
txscript.MultisigRedeemScriptFromScriptSig(
txIn.SignatureScript)
if err != nil {
continue
}
class, addrs, _, err := txscript.ExtractPkScriptAddrs(
txscript.DefaultScriptVersion, rs, idx.chainParams)
if err != nil {
// Non-standard outputs are skipped.
continue
}
if class != txscript.MultiSigTy {
// This should never happen, but be paranoid.
continue
}
for _, addr := range addrs {
k, err := addrToKey(addr, idx.chainParams)
if err != nil {
continue
}
usedAddrs[k] = struct{}{}
}
}
}
for _, txOut := range tx.MsgTx().TxOut {
class, addrs, _, err := txscript.ExtractPkScriptAddrs(
txOut.Version, txOut.PkScript, idx.chainParams)
if err != nil {
// Non-standard outputs are skipped.
continue
}
if isSStx && class == txscript.NullDataTy {
addr, err := stake.AddrFromSStxPkScrCommitment(txOut.PkScript,
idx.chainParams)
if err != nil {
// Ignore unsupported address types.
continue
}
addrs = append(addrs, addr)
}
for _, addr := range addrs {
k, err := addrToKey(addr, idx.chainParams)
if err != nil {
// Ignore unsupported address types.
continue
}
usedAddrs[k] = struct{}{}
}
}
}
// Write all the newly used addresses to the database,
// skipping any keys that already exist. Write any
// addresses we see in mempool at this time, too,
// then remove them from the unconfirmed map drop
// dropping the old map and reassigning a new map.
idx.unconfirmedLock.Lock()
for k := range idx.mpExistsAddr {
usedAddrs[k] = struct{}{}
}
idx.mpExistsAddr = make(map[[addrKeySize]byte]struct{})
idx.unconfirmedLock.Unlock()
meta := dbTx.Metadata()
existsAddrIndex := meta.Bucket(existsAddrIndexKey)
newUsedAddrs := make(map[[addrKeySize]byte]struct{})
for k := range usedAddrs {
if !idx.existsAddress(existsAddrIndex, k) {
newUsedAddrs[k] = struct{}{}
}
}
for k := range newUsedAddrs {
err := dbPutExistsAddr(existsAddrIndex, k)
if err != nil {
return err
}
}
return nil
}
// dbPutTxIndexEntry uses an existing database transaction to update the
// transaction index given the provided serialized data that is expected to have
// been serialized putTxIndexEntry.
func dbPutTxIndexEntry(dbTx database.Tx, txHash chainhash.Hash, serializedData []byte) error {
txIndex := dbTx.Metadata().Bucket(txIndexKey)
return txIndex.Put(txHash[:], serializedData)
}
// Create is invoked when the indexer manager determines the index needs
// to be created for the first time. It creates the bucket for the address
// index.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) Create(dbTx database.Tx) error {
_, err := dbTx.Metadata().CreateBucket(existsAddrIndexKey)
return err
}