// testNewestShaEmpty ensures the NewestSha returns the values expected by
// the interface contract.
func testNewestShaEmpty(t *testing.T, db btcdb.Db) {
sha, height, err := db.NewestSha()
if err != nil {
t.Errorf("NewestSha error %v", err)
}
if !sha.IsEqual(&zeroHash) {
t.Errorf("NewestSha wrong hash got: %s, want %s", sha, &zeroHash)
}
if height != -1 {
t.Errorf("NewestSha wrong height got: %d, want %d", height, -1)
}
}
func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) {
var testincrement int64 = 50
var testcnt int64 = 100
shanames := make([]*btcwire.ShaHash, len(blocks))
nBlocks := int64(len(blocks))
for i := range blocks {
blockSha, err := blocks[i].Sha()
if err != nil {
t.Errorf("FetchRangeHeight: unexpected failure computing block sah %v", err)
}
shanames[i] = blockSha
}
for startheight := int64(0); startheight < nBlocks; startheight += testincrement {
endheight := startheight + testcnt
if endheight > nBlocks {
endheight = btcdb.AllShas
}
shalist, err := db.FetchHeightRange(startheight, endheight)
if err != nil {
t.Errorf("FetchRangeHeight: unexpected failure looking up shas %v", err)
}
if endheight == btcdb.AllShas {
if int64(len(shalist)) != nBlocks-startheight {
t.Errorf("FetchRangeHeight: expected A %v shas, got %v", nBlocks-startheight, len(shalist))
}
} else {
if int64(len(shalist)) != testcnt {
t.Errorf("FetchRangeHeight: expected %v shas, got %v", testcnt, len(shalist))
}
}
for i := range shalist {
sha0 := *shanames[int64(i)+startheight]
sha1 := shalist[i]
if sha0 != sha1 {
t.Errorf("FetchRangeHeight: mismatch sha at %v requested range %v %v: %v %v ", int64(i)+startheight, startheight, endheight, sha0, sha1)
}
}
}
}
func testIterator(t *testing.T, db btcdb.Db, shas []btcwire.ShaHash,
sync string) {
// Iterate over the whole list of shas.
iter, err := db.NewIterateBlocks()
if err != nil {
t.Errorf("failed to create iterated blocks")
return
}
// Skip the genesis block.
_ = iter.NextRow()
i := 0
for ; iter.NextRow(); i++ {
key, pver, buf, err := iter.Row()
if err != nil {
t.Errorf("iter.NextRow() failed: %v (%s)", err, sync)
break
}
if i >= len(shas) {
t.Errorf("iterator returned more shas than "+
"expected - %d (%s)", i, sync)
break
}
if !key.IsEqual(&shas[i]) {
t.Errorf("iterator test: %dth sha doesn't match (%s)",
i, sync)
}
if !bytes.Equal(zeroBlock, buf) {
t.Errorf("iterator test: %d buf incorrect (%s)", i,
sync)
}
if pver != 1 {
t.Errorf("iterator: %dth pver is %d and not 1 (%s)",
i, pver, sync)
}
}
if i < len(shas) {
t.Errorf("iterator got no rows on %dth loop, should have %d "+
"(%s)", i, len(shas), sync)
}
if _, _, _, err = iter.Row(); err == nil {
t.Errorf("done iterator didn't return failure")
}
iter.Close()
}
func testNewestSha(t *testing.T, db btcdb.Db, expSha btcwire.ShaHash,
expBlk int64, situation string) {
newestsha, blkid, err := db.NewestSha()
if err != nil {
t.Errorf("NewestSha failed %v (%s)", err, situation)
return
}
if blkid != expBlk {
t.Errorf("NewestSha blkid is %d not %d (%s)", blkid, expBlk,
situation)
}
if !newestsha.IsEqual(&expSha) {
t.Errorf("Newestsha isn't the last sha we inserted %v %v (%s)",
newestsha, &expSha, situation)
}
}
func DumpBlock(db btcdb.Db, height int64, fo io.Writer, rflag bool, fflag bool, tflag bool) error {
sha, err := db.FetchBlockShaByHeight(height)
if err != nil {
return err
}
blk, err := db.FetchBlockBySha(sha)
if err != nil {
log.Warnf("Failed to fetch block %v, err %v", sha, err)
return err
}
rblk, err := blk.Bytes()
blkid := blk.Height()
if rflag {
log.Infof("Block %v depth %v %v", sha, blkid, spew.Sdump(rblk))
}
mblk := blk.MsgBlock()
if fflag {
log.Infof("Block %v depth %v %v", sha, blkid, spew.Sdump(mblk))
}
if tflag {
log.Infof("Num transactions %v", len(mblk.Transactions))
for i, tx := range mblk.Transactions {
txsha, err := tx.TxSha()
if err != nil {
continue
}
log.Infof("tx %v: %v", i, &txsha)
}
}
if fo != nil {
// generate and write header values
binary.Write(fo, binary.LittleEndian, uint32(btcwire.MainNet))
binary.Write(fo, binary.LittleEndian, uint32(len(rblk)))
// write block
fo.Write(rblk)
}
return nil
}
func getHeight(db btcdb.Db, str string) (int64, error) {
argtype, idx, sha, err := parsesha(str)
if err != nil {
log.Warnf("unable to decode [%v] %v", str, err)
return 0, err
}
switch argtype {
case ArgSha:
// nothing to do
blk, err := db.FetchBlockBySha(sha)
if err != nil {
log.Warnf("unable to locate block sha %v err %v",
sha, err)
return 0, err
}
idx = blk.Height()
case ArgHeight:
}
return idx, nil
}
func getSha(db btcdb.Db, str string) (btcwire.ShaHash, error) {
argtype, idx, sha, err := parsesha(str)
if err != nil {
log.Warnf("unable to decode [%v] %v", str, err)
return btcwire.ShaHash{}, err
}
switch argtype {
case ArgSha:
// nothing to do
case ArgHeight:
sha, err = db.FetchBlockShaByHeight(idx)
if err != nil {
return btcwire.ShaHash{}, err
}
}
if sha == nil {
fmt.Printf("wtf sha is nil but err is %v", err)
}
return *sha, nil
}
func fetch(db btcdb.Db, rd *rData) error {
sha, err := db.FetchBlockShaByHeight(rd.in.H)
if err != nil {
return fmt.Errorf("failed FetchBlockShaByHeight(%v): %v\n", rd.in.H, err)
}
blk, err := db.FetchBlockBySha(sha)
if err != nil {
return fmt.Errorf("failed FetchBlockBySha(%v) - h %v: %v\n", sha, rd.in.H, err)
}
tx := blk.Transactions()[rd.in.Tx]
rd.blkSha = sha
rd.blk = blk
rd.tx = tx
rd.txInIndex = rd.in.TxIn
rd.txIn = tx.MsgTx().TxIn[rd.in.TxIn]
txPrevList, err := db.FetchTxBySha(&rd.txIn.PreviousOutPoint.Hash)
if err != nil {
return fmt.Errorf("failed FetchTxBySha(%v) - h %v: %v\n",
rd.txIn.PreviousOutPoint.Hash, rd.in.H, err)
}
if len(txPrevList) != 1 {
return fmt.Errorf("not single FetchTxBySha(%v) - h %v: %v\n",
rd.txIn.PreviousOutPoint.Hash, rd.in.H, len(txPrevList))
}
blkPrev, err := db.FetchBlockBySha(txPrevList[0].BlkSha)
if err != nil {
return fmt.Errorf("failed prev FetchBlockBySha(%v) - h %v: %v\n",
txPrevList[0].BlkSha, rd.in.H, err)
}
rd.txPrev = txPrevList[0]
rd.txPrevOutIndex = rd.txIn.PreviousOutPoint.Index
rd.txPrevOut = rd.txPrev.Tx.TxOut[rd.txPrevOutIndex]
rd.blkPrev = blkPrev
return nil
}
func verifyChain(db btcdb.Db, level, depth int32) error {
_, curheight64, err := db.NewestSha()
if err != nil {
rpcsLog.Errorf("Verify is unable to fetch current block "+
"height: %v", err)
}
curheight := int32(curheight64)
if depth > curheight {
depth = curheight
}
for height := curheight; height > (curheight - depth); height-- {
// Level 0 just looks up the block.
sha, err := db.FetchBlockShaByHeight(int64(height))
if err != nil {
rpcsLog.Errorf("Verify is unable to fetch block at "+
"height %d: %v", height, err)
return err
}
block, err := db.FetchBlockBySha(sha)
if err != nil {
rpcsLog.Errorf("Verify is unable to fetch block at "+
"sha %v height %d: %v", sha, height, err)
return err
}
// Level 1 does basic chain sanity checks.
if level > 0 {
err := btcchain.CheckBlockSanity(block,
activeNetParams.powLimit)
if err != nil {
rpcsLog.Errorf("Verify is unable to "+
"validate block at sha %v height "+
"%d: %v", sha, height, err)
return err
}
}
}
rpcsLog.Infof("Chain verify completed successfully")
return nil
}
func FindSender(txIns []*btcwire.TxIn, btcdb btcdb.Db) (Address, error) {
inputs := make(map[string]int64)
for _, txIn := range txIns {
op := txIn.PreviousOutpoint
hash := op.Hash
index := op.Index
transactions, err := btcdb.FetchTxBySha(&hash)
if err != nil {
return Address{}, err
}
// TODO: During initial sync unconfirmed transactions might be picked up
// We should prevent that from showing up but this is a work around
// When a transaction is not in the database yet
if len(transactions) == 0 {
continue
}
previousOutput := transactions[0].Tx.TxOut[index]
// The largest contributor receives the Mastercoins, so add multiple address values together
address, _ := GetAddrs(previousOutput.PkScript)
inputs[address[0].Addr] += previousOutput.Value
}
// Decide which input has the most value so we know who is sending this transaction
var highest int64
var highestAddress string
for k, v := range inputs {
if v > highest {
highest = v
highestAddress = k
}
}
return Address{Addr: highestAddress}, nil
}
// findCandidates searches the chain backwards for checkpoint candidates and
// returns a slice of found candidates, if any. It also stops searching for
// candidates at the last checkpoint that is already hard coded into btcchain
// since there is no point in finding candidates before already existing
// checkpoints.
func findCandidates(db btcdb.Db, latestHash *btcwire.ShaHash) ([]*btcchain.Checkpoint, error) {
// Start with the latest block of the main chain.
block, err := db.FetchBlockBySha(latestHash)
if err != nil {
return nil, err
}
// Setup chain and get the latest checkpoint. Ignore notifications
// since they aren't needed for this util.
chain := btcchain.New(db, activeNetwork, nil)
latestCheckpoint := chain.LatestCheckpoint()
if latestCheckpoint == nil {
return nil, fmt.Errorf("unable to retrieve latest checkpoint")
}
// The latest known block must be at least the last known checkpoint
// plus required checkpoint confirmations.
checkpointConfirmations := int64(btcchain.CheckpointConfirmations)
requiredHeight := latestCheckpoint.Height + checkpointConfirmations
if block.Height() < requiredHeight {
return nil, fmt.Errorf("the block database is only at height "+
"%d which is less than the latest checkpoint height "+
"of %d plus required confirmations of %d",
block.Height(), latestCheckpoint.Height,
checkpointConfirmations)
}
// Indeterminate progress setup.
numBlocksToTest := block.Height() - requiredHeight
progressInterval := (numBlocksToTest / 100) + 1 // min 1
fmt.Print("Searching for candidates")
defer fmt.Println()
// Loop backwards through the chain to find checkpoint candidates.
var candidates []*btcchain.Checkpoint
numTested := int64(0)
for len(candidates) < cfg.NumCandidates && block.Height() > requiredHeight {
// Display progress.
if numTested%progressInterval == 0 {
fmt.Print(".")
}
// Determine if this block is a checkpoint candidate.
isCandidate, err := chain.IsCheckpointCandidate(block)
if err != nil {
return nil, err
}
// All checks passed, so this node seems like a reasonable
// checkpoint candidate.
if isCandidate {
candidateHash, err := block.Sha()
if err != nil {
return nil, err
}
checkpoint := btcchain.Checkpoint{
Height: block.Height(),
Hash: candidateHash,
}
candidates = append(candidates, &checkpoint)
}
prevHash := &block.MsgBlock().Header.PrevBlock
block, err = db.FetchBlockBySha(prevHash)
if err != nil {
return nil, err
}
numTested++
}
return candidates, nil
}
// chainSetup is used to create a new db and chain instance with the genesis
// block already inserted. In addition to the new chain instnce, it returns
// a teardown function the caller should invoke when done testing to clean up.
func chainSetup(dbName string) (*btcchain.BlockChain, func(), error) {
if !isSupportedDbType(testDbType) {
return nil, nil, fmt.Errorf("unsupported db type %v", testDbType)
}
// Handle memory database specially since it doesn't need the disk
// specific handling.
var db btcdb.Db
var teardown func()
if testDbType == "memdb" {
ndb, err := btcdb.CreateDB(testDbType)
if err != nil {
return nil, nil, fmt.Errorf("error creating db: %v", err)
}
db = ndb
// Setup a teardown function for cleaning up. This function is
// returned to the caller to be invoked when it is done testing.
teardown = func() {
db.Close()
}
} else {
// Create the root directory for test databases.
if !fileExists(testDbRoot) {
if err := os.MkdirAll(testDbRoot, 0700); err != nil {
err := fmt.Errorf("unable to create test db "+
"root: %v", err)
return nil, nil, err
}
}
// Create a new database to store the accepted blocks into.
dbPath := filepath.Join(testDbRoot, dbName)
_ = os.RemoveAll(dbPath)
ndb, err := btcdb.CreateDB(testDbType, dbPath)
if err != nil {
return nil, nil, fmt.Errorf("error creating db: %v", err)
}
db = ndb
// Setup a teardown function for cleaning up. This function is
// returned to the caller to be invoked when it is done testing.
teardown = func() {
dbVersionPath := filepath.Join(testDbRoot, dbName+".ver")
db.Sync()
db.Close()
os.RemoveAll(dbPath)
os.Remove(dbVersionPath)
os.RemoveAll(testDbRoot)
}
}
// Insert the main network genesis block. This is part of the initial
// database setup.
genesisBlock := btcutil.NewBlock(&btcwire.GenesisBlock)
_, err := db.InsertBlock(genesisBlock)
if err != nil {
teardown()
err := fmt.Errorf("failed to insert genesis block: %v", err)
return nil, nil, err
}
chain := btcchain.New(db, btcwire.MainNet, nil)
return chain, teardown, nil
}
func getSignatures(maxHeigth int64, log btclog.Logger, db btcdb.Db) chan *rData {
heigthChan := make(chan int64)
blockChan := make(chan *btcutil.Block)
sigChan := make(chan *rData)
go func() {
for h := int64(0); h < maxHeigth; h++ {
heigthChan <- h
}
close(heigthChan)
}()
var blockWg sync.WaitGroup
for i := 0; i <= 10; i++ {
blockWg.Add(1)
go func() {
for h := range heigthChan {
sha, err := db.FetchBlockShaByHeight(h)
if err != nil {
log.Warnf("failed FetchBlockShaByHeight(%v): %v", h, err)
return
}
blk, err := db.FetchBlockBySha(sha)
if err != nil {
log.Warnf("failed FetchBlockBySha(%v) - h %v: %v", sha, h, err)
return
}
blockChan <- blk
}
blockWg.Done()
}()
}
go func() {
blockWg.Wait()
close(blockChan)
}()
var sigWg sync.WaitGroup
for i := 0; i <= 10; i++ {
sigWg.Add(1)
go func() {
for blk := range blockChan {
mblk := blk.MsgBlock()
for i, tx := range mblk.Transactions {
if btcchain.IsCoinBase(btcutil.NewTx(tx)) {
continue
}
for t, txin := range tx.TxIn {
dataSlice, err := btcscript.PushedData(txin.SignatureScript)
if err != nil {
continue
}
for d, data := range dataSlice {
signature, err := btcec.ParseSignature(data, btcec.S256())
if err != nil {
continue
}
sigChan <- &rData{
sig: signature,
H: blk.Height(),
Tx: i,
TxIn: t,
Data: d,
}
}
}
}
}
sigWg.Done()
}()
}
go func() {
sigWg.Wait()
close(sigChan)
}()
return sigChan
}
func search(log btclog.Logger, db btcdb.Db) map[string][]*rData {
// Setup signal handler
signalChan := make(chan os.Signal, 1)
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR1)
// Potential optimisation: keep the bloom filter between runs
filter := dablooms.NewScalingBloom(bloomSize, bloomRate, "blockchainr_bloom.bin")
if filter == nil {
log.Warn("dablooms.NewScalingBloom failed")
return nil
}
potentialValues := make(stringSet)
rMap := make(map[string][]*rData)
_, maxHeigth, err := db.NewestSha()
if err != nil {
log.Warnf("db NewestSha failed: %v", err)
return nil
}
for step := 1; step <= 2; step++ {
lastTime := time.Now()
lastSig := int64(0)
sigCounter := int64(0)
matches := int64(0)
ticker := time.Tick(tickFreq * time.Second)
signatures := getSignatures(maxHeigth, log, db)
for rd := range signatures {
select {
case s := <-signalChan:
log.Infof("Step %v - signal %v - %v sigs in %.2fs, %v matches, %v total, block %v of %v",
step, s, sigCounter-lastSig, time.Since(lastTime).Seconds(),
matches, sigCounter, rd.H, maxHeigth)
if s == syscall.SIGINT || s == syscall.SIGTERM {
return rMap
}
case <-ticker:
log.Infof("Step %v - %v sigs in %.2fs, %v matches, %v total, block %v of %v",
step, sigCounter-lastSig, time.Since(lastTime).Seconds(),
matches, sigCounter, rd.H, maxHeigth)
lastTime = time.Now()
lastSig = sigCounter
default:
break
}
// Potential optimisation: store in potentialValues also the block
// height, and if step 2 finds the same h first, it's a bloom
// false positive
if step == 1 {
b := rd.sig.R.Bytes()
if filter.Check(b) {
matches++
potentialValues.Add(rd.sig.R.String())
} else {
if !filter.Add(b, 1) {
log.Warn("Add failed (?)")
}
}
} else if step == 2 {
if potentialValues.Contains(rd.sig.R.String()) {
matches++
rMap[rd.sig.R.String()] = append(rMap[rd.sig.R.String()], rd)
}
}
sigCounter++
}
if *memprofile != "" {
f, err := os.Create(fmt.Sprintf("%s.%d", *memprofile, step))
if err != nil {
log.Warnf("open memprofile failed: %v", err)
return nil
}
pprof.WriteHeapProfile(f)
f.Close()
}
log.Infof("Step %v done - %v signatures processed - %v matches",
step, sigCounter, matches)
}
return rMap
}
func testFetch(t *testing.T, db btcdb.Db, shas []btcwire.ShaHash,
sync string) {
// Test the newest sha is what we expect and call it twice to ensure
// caching is working working properly.
numShas := int64(len(shas))
newestSha := shas[numShas-1]
newestBlockID := int64(numShas)
testNewestSha(t, db, newestSha, newestBlockID, sync)
testNewestSha(t, db, newestSha, newestBlockID, sync+" cached")
for i, sha := range shas {
// Add one for genesis block skew.
i = i + 1
// Ensure the sha exists in the db as expected.
if !db.ExistsSha(&sha) {
t.Errorf("testSha %d doesn't exists (%s)", i, sync)
break
}
// Fetch the sha from the db and ensure all fields are expected
// values.
buf, pver, idx, err := sqlite3.FetchSha(db, &sha)
if err != nil {
t.Errorf("Failed to fetch testSha %d (%s)", i, sync)
}
if !bytes.Equal(zeroBlock, buf) {
t.Errorf("testSha %d incorrect block return (%s)", i,
sync)
}
if pver != 1 {
t.Errorf("pver is %d and not 1 for testSha %d (%s)",
pver, i, sync)
}
if idx != int64(i) {
t.Errorf("index isn't as expected %d vs %d (%s)",
idx, i, sync)
}
// Fetch the sha by index and ensure it matches.
tsha, err := db.FetchBlockShaByHeight(int64(i))
if err != nil {
t.Errorf("can't fetch sha at index %d: %v", i, err)
continue
}
if !tsha.IsEqual(&sha) {
t.Errorf("sha for index %d isn't shas[%d]", i, i)
}
}
endBlockID := numShas + 1
midBlockID := endBlockID / 2
fetchIdxTests := []fetchIdxTest{
// All shas.
{1, btcdb.AllShas, shas, "fetch all shas"},
//// All shas using known bounds.
{1, endBlockID, shas, "fetch all shas2"},
// Partial list starting at beginning.
{1, midBlockID, shas[:midBlockID-1], "fetch first half"},
// Partial list ending at end.
{midBlockID, endBlockID, shas[midBlockID-1 : endBlockID-1],
"fetch second half"},
// Nonexistent off the end.
{endBlockID, endBlockID * 2, []btcwire.ShaHash{},
"fetch nonexistent"},
}
for _, test := range fetchIdxTests {
t.Logf("numSha: %d - Fetch from %d to %d\n", numShas, test.start, test.end)
if shalist, err := db.FetchHeightRange(test.start, test.end); err == nil {
compareArray(t, shalist, test.exp, test.test, sync)
} else {
t.Errorf("failed to fetch index range for %s (%s)",
test.test, sync)
}
}
// Try and fetch nonexistent sha.
if db.ExistsSha(&badSha) {
t.Errorf("nonexistent sha exists (%s)!", sync)
}
_, _, _, err := sqlite3.FetchSha(db, &badSha)
if err == nil {
t.Errorf("Success when fetching a bad sha! (%s)", sync)
}
// XXX if not check to see it is the right value?
testIterator(t, db, shas, sync)
}