// AssertChannelExists asserts that an active channel identified by
// channelPoint is known to exist from the point-of-view of node..
func (n *networkHarness) AssertChannelExists(ctx context.Context,
node *lightningNode, chanPoint *wire.OutPoint) error {
req := &lnrpc.ListChannelsRequest{}
resp, err := node.ListChannels(ctx, req)
if err != nil {
return fmt.Errorf("unable fetch node's channels: %v", err)
}
for _, channel := range resp.Channels {
if channel.ChannelPoint == chanPoint.String() {
return nil
}
}
return fmt.Errorf("channel not found")
}
func readCanonicalOutPoint(k []byte, op *wire.OutPoint) error {
if len(k) < 36 {
str := "short canonical outpoint"
return storeError(ErrData, str, nil)
}
copy(op.Hash[:], k)
op.Index = byteOrder.Uint32(k[32:36])
return nil
}
// removeOrphan is the internal function which implements the public
// RemoveOrphan. See the comment for RemoveOrphan for more details.
//
// This function MUST be called with the mempool lock held (for writes).
func (mp *TxPool) removeOrphan(tx *btcutil.Tx, removeRedeemers bool) {
// Nothing to do if passed tx is not an orphan.
txHash := tx.Hash()
otx, exists := mp.orphans[*txHash]
if !exists {
return
}
// Remove the reference from the previous orphan index.
for _, txIn := range otx.tx.MsgTx().TxIn {
orphans, exists := mp.orphansByPrev[txIn.PreviousOutPoint]
if exists {
delete(orphans, *txHash)
// Remove the map entry altogether if there are no
// longer any orphans which depend on it.
if len(orphans) == 0 {
delete(mp.orphansByPrev, txIn.PreviousOutPoint)
}
}
}
// Remove any orphans that redeem outputs from this one if requested.
if removeRedeemers {
prevOut := wire.OutPoint{Hash: *txHash}
for txOutIdx := range tx.MsgTx().TxOut {
prevOut.Index = uint32(txOutIdx)
for _, orphan := range mp.orphansByPrev[prevOut] {
mp.removeOrphan(orphan, true)
}
}
}
// Remove the transaction from the orphan pool.
delete(mp.orphans, *txHash)
}
func readOutpoint(r io.Reader, o *wire.OutPoint) error {
scratch := make([]byte, 4)
txid, err := wire.ReadVarBytes(r, 0, 32, "prevout")
if err != nil {
return err
}
copy(o.Hash[:], txid)
if _, err := r.Read(scratch); err != nil {
return err
}
o.Index = byteOrder.Uint32(scratch)
return nil
}
// Ingest puts a tx into the DB atomically. This can result in a
// gain, a loss, or no result. Gain or loss in satoshis is returned.
func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
var hits uint32
var err error
var nUtxoBytes [][]byte
// tx has been OK'd by SPV; check tx sanity
utilTx := btcutil.NewTx(tx) // convert for validation
// checks basic stuff like there are inputs and ouputs
err = blockchain.CheckTransactionSanity(utilTx)
if err != nil {
return hits, err
}
// note that you can't check signatures; this is SPV.
// 0 conf SPV means pretty much nothing. Anyone can say anything.
spentOPs := make([][]byte, len(tx.TxIn))
// before entering into db, serialize all inputs of the ingested tx
for i, txin := range tx.TxIn {
spentOPs[i], err = outPointToBytes(&txin.PreviousOutPoint)
if err != nil {
return hits, err
}
}
// go through txouts, and then go through addresses to match
// generate PKscripts for all addresses
wPKscripts := make([][]byte, len(ts.Adrs))
aPKscripts := make([][]byte, len(ts.Adrs))
for i, _ := range ts.Adrs {
// iterate through all our addresses
// convert regular address to witness address. (split adrs later)
wa, err := btcutil.NewAddressWitnessPubKeyHash(
ts.Adrs[i].PkhAdr.ScriptAddress(), ts.Param)
if err != nil {
return hits, err
}
wPKscripts[i], err = txscript.PayToAddrScript(wa)
if err != nil {
return hits, err
}
aPKscripts[i], err = txscript.PayToAddrScript(ts.Adrs[i].PkhAdr)
if err != nil {
return hits, err
}
}
cachedSha := tx.TxSha()
// iterate through all outputs of this tx, see if we gain
for i, out := range tx.TxOut {
for j, ascr := range aPKscripts {
// detect p2wpkh
witBool := false
if bytes.Equal(out.PkScript, wPKscripts[j]) {
witBool = true
}
if bytes.Equal(out.PkScript, ascr) || witBool { // new utxo found
var newu Utxo // create new utxo and copy into it
newu.AtHeight = height
newu.KeyIdx = ts.Adrs[j].KeyIdx
newu.Value = out.Value
newu.IsWit = witBool // copy witness version from pkscript
var newop wire.OutPoint
newop.Hash = cachedSha
newop.Index = uint32(i)
newu.Op = newop
b, err := newu.ToBytes()
if err != nil {
return hits, err
}
nUtxoBytes = append(nUtxoBytes, b)
hits++
break // txos can match only 1 script
}
}
}
err = ts.StateDB.Update(func(btx *bolt.Tx) error {
// get all 4 buckets
duf := btx.Bucket(BKTUtxos)
// sta := btx.Bucket(BKTState)
old := btx.Bucket(BKTStxos)
txns := btx.Bucket(BKTTxns)
if duf == nil || old == nil || txns == nil {
return fmt.Errorf("error: db not initialized")
}
// iterate through duffel bag and look for matches
// this makes us lose money, which is regrettable, but we need to know.
for _, nOP := range spentOPs {
v := duf.Get(nOP)
if v != nil {
hits++
// do all this just to figure out value we lost
x := make([]byte, len(nOP)+len(v))
copy(x, nOP)
copy(x[len(nOP):], v)
lostTxo, err := UtxoFromBytes(x)
if err != nil {
return err
}
// after marking for deletion, save stxo to old bucket
var st Stxo // generate spent txo
st.Utxo = lostTxo // assign outpoint
st.SpendHeight = height // spent at height
st.SpendTxid = cachedSha // spent by txid
stxb, err := st.ToBytes() // serialize
if err != nil {
return err
}
err = old.Put(nOP, stxb) // write nOP:v outpoint:stxo bytes
if err != nil {
return err
}
err = duf.Delete(nOP)
if err != nil {
return err
}
}
}
// done losing utxos, next gain utxos
// next add all new utxos to db, this is quick as the work is above
for _, ub := range nUtxoBytes {
err = duf.Put(ub[:36], ub[36:])
if err != nil {
return err
}
}
// if hits is nonzero it's a relevant tx and we should store it
var buf bytes.Buffer
tx.Serialize(&buf)
err = txns.Put(cachedSha.Bytes(), buf.Bytes())
if err != nil {
return err
}
return nil
})
return hits, err
}
// processOrphans is the internal function which implements the public
// ProcessOrphans. See the comment for ProcessOrphans for more details.
//
// This function MUST be called with the mempool lock held (for writes).
func (mp *TxPool) processOrphans(acceptedTx *btcutil.Tx) []*TxDesc {
var acceptedTxns []*TxDesc
// Start with processing at least the passed transaction.
processList := list.New()
processList.PushBack(acceptedTx)
for processList.Len() > 0 {
// Pop the transaction to process from the front of the list.
firstElement := processList.Remove(processList.Front())
processItem := firstElement.(*btcutil.Tx)
prevOut := wire.OutPoint{Hash: *processItem.Hash()}
for txOutIdx := range processItem.MsgTx().TxOut {
// Look up all orphans that redeem the output that is
// now available. This will typically only be one, but
// it could be multiple if the orphan pool contains
// double spends. While it may seem odd that the orphan
// pool would allow this since there can only possibly
// ultimately be a single redeemer, it's important to
// track it this way to prevent malicious actors from
// being able to purposely constructing orphans that
// would otherwise make outputs unspendable.
//
// Skip to the next available output if there are none.
prevOut.Index = uint32(txOutIdx)
orphans, exists := mp.orphansByPrev[prevOut]
if !exists {
continue
}
// Potentially accept an orphan into the tx pool.
for _, tx := range orphans {
missing, txD, err := mp.maybeAcceptTransaction(
tx, true, true, false)
if err != nil {
// The orphan is now invalid, so there
// is no way any other orphans which
// redeem any of its outputs can be
// accepted. Remove them.
mp.removeOrphan(tx, true)
break
}
// Transaction is still an orphan. Try the next
// orphan which redeems this output.
if len(missing) > 0 {
continue
}
// Transaction was accepted into the main pool.
//
// Add it to the list of accepted transactions
// that are no longer orphans, remove it from
// the orphan pool, and add it to the list of
// transactions to process so any orphans that
// depend on it are handled too.
acceptedTxns = append(acceptedTxns, txD)
mp.removeOrphan(tx, false)
processList.PushBack(tx)
// Only one transaction for this outpoint can be
// accepted, so the rest are now double spends
// and are removed later.
break
}
}
}
// Recursively remove any orphans that also redeem any outputs redeemed
// by the accepted transactions since those are now definitive double
// spends.
mp.removeOrphanDoubleSpends(acceptedTx)
for _, txD := range acceptedTxns {
mp.removeOrphanDoubleSpends(txD.Tx)
}
return acceptedTxns
}
func (s *Store) rollback(ns walletdb.Bucket, height int32) error {
minedBalance, err := fetchMinedBalance(ns)
if err != nil {
return err
}
// Keep track of all credits that were removed from coinbase
// transactions. After detaching all blocks, if any transaction record
// exists in unmined that spends these outputs, remove them and their
// spend chains.
//
// It is necessary to keep these in memory and fix the unmined
// transactions later since blocks are removed in increasing order.
var coinBaseCredits []wire.OutPoint
it := makeBlockIterator(ns, height)
for it.next() {
b := &it.elem
log.Infof("Rolling back %d transactions from block %v height %d",
len(b.transactions), b.Hash, b.Height)
for i := range b.transactions {
txHash := &b.transactions[i]
recKey := keyTxRecord(txHash, &b.Block)
recVal := existsRawTxRecord(ns, recKey)
var rec TxRecord
err = readRawTxRecord(txHash, recVal, &rec)
if err != nil {
return err
}
err = deleteTxRecord(ns, txHash, &b.Block)
if err != nil {
return err
}
// Handle coinbase transactions specially since they are
// not moved to the unconfirmed store. A coinbase cannot
// contain any debits, but all credits should be removed
// and the mined balance decremented.
if blockchain.IsCoinBaseTx(&rec.MsgTx) {
op := wire.OutPoint{Hash: rec.Hash}
for i, output := range rec.MsgTx.TxOut {
k, v := existsCredit(ns, &rec.Hash,
uint32(i), &b.Block)
if v == nil {
continue
}
op.Index = uint32(i)
coinBaseCredits = append(coinBaseCredits, op)
unspentKey, credKey := existsUnspent(ns, &op)
if credKey != nil {
minedBalance -= btcutil.Amount(output.Value)
err = deleteRawUnspent(ns, unspentKey)
if err != nil {
return err
}
}
err = deleteRawCredit(ns, k)
if err != nil {
return err
}
}
continue
}
err = putRawUnmined(ns, txHash[:], recVal)
if err != nil {
return err
}
// For each debit recorded for this transaction, mark
// the credit it spends as unspent (as long as it still
// exists) and delete the debit. The previous output is
// recorded in the unconfirmed store for every previous
// output, not just debits.
for i, input := range rec.MsgTx.TxIn {
prevOut := &input.PreviousOutPoint
prevOutKey := canonicalOutPoint(&prevOut.Hash,
prevOut.Index)
err = putRawUnminedInput(ns, prevOutKey, rec.Hash[:])
if err != nil {
return err
}
// If this input is a debit, remove the debit
// record and mark the credit that it spent as
// unspent, incrementing the mined balance.
debKey, credKey, err := existsDebit(ns,
&rec.Hash, uint32(i), &b.Block)
if err != nil {
return err
}
if debKey == nil {
continue
}
// unspendRawCredit does not error in case the
// no credit exists for this key, but this
// behavior is correct. Since blocks are
// removed in increasing order, this credit
// may have already been removed from a
// previously removed transaction record in
// this rollback.
var amt btcutil.Amount
amt, err = unspendRawCredit(ns, credKey)
if err != nil {
return err
}
err = deleteRawDebit(ns, debKey)
if err != nil {
return err
}
// If the credit was previously removed in the
// rollback, the credit amount is zero. Only
// mark the previously spent credit as unspent
// if it still exists.
if amt == 0 {
continue
}
unspentVal, err := fetchRawCreditUnspentValue(credKey)
if err != nil {
return err
}
minedBalance += amt
err = putRawUnspent(ns, prevOutKey, unspentVal)
if err != nil {
return err
}
}
// For each detached non-coinbase credit, move the
// credit output to unmined. If the credit is marked
// unspent, it is removed from the utxo set and the
// mined balance is decremented.
//
// TODO: use a credit iterator
for i, output := range rec.MsgTx.TxOut {
k, v := existsCredit(ns, &rec.Hash, uint32(i),
&b.Block)
if v == nil {
continue
}
amt, change, err := fetchRawCreditAmountChange(v)
if err != nil {
return err
}
outPointKey := canonicalOutPoint(&rec.Hash, uint32(i))
unminedCredVal := valueUnminedCredit(amt, change)
err = putRawUnminedCredit(ns, outPointKey, unminedCredVal)
if err != nil {
return err
}
err = deleteRawCredit(ns, k)
if err != nil {
return err
}
credKey := existsRawUnspent(ns, outPointKey)
if credKey != nil {
minedBalance -= btcutil.Amount(output.Value)
err = deleteRawUnspent(ns, outPointKey)
if err != nil {
return err
}
}
}
}
err = it.delete()
if err != nil {
return err
}
}
if it.err != nil {
return it.err
}
for _, op := range coinBaseCredits {
opKey := canonicalOutPoint(&op.Hash, op.Index)
unminedKey := existsRawUnminedInput(ns, opKey)
if unminedKey != nil {
unminedVal := existsRawUnmined(ns, unminedKey)
var unminedRec TxRecord
copy(unminedRec.Hash[:], unminedKey) // Silly but need an array
err = readRawTxRecord(&unminedRec.Hash, unminedVal, &unminedRec)
if err != nil {
return err
}
log.Debugf("Transaction %v spends a removed coinbase "+
"output -- removing as well", unminedRec.Hash)
err = s.removeConflict(ns, &unminedRec)
if err != nil {
return err
}
}
}
return putMinedBalance(ns, minedBalance)
}
func testMultiHopPayments(net *networkHarness, t *harnessTest) {
const chanAmt = btcutil.Amount(100000)
ctxb := context.Background()
timeout := time.Duration(time.Second * 5)
// Open a channel with 100k satoshis between Alice and Bob with Alice
// being the sole funder of the channel.
ctxt, _ := context.WithTimeout(ctxb, timeout)
chanPointAlice := openChannelAndAssert(t, net, ctxt, net.Alice,
net.Bob, chanAmt)
aliceChanTXID, err := wire.NewShaHash(chanPointAlice.FundingTxid)
if err != nil {
t.Fatalf("unable to create sha hash: %v", err)
}
aliceFundPoint := wire.OutPoint{
Hash: *aliceChanTXID,
Index: chanPointAlice.OutputIndex,
}
// Create a new node (Carol), load her with some funds, then establish
// a connection between Carol and Alice with a channel that has
// identical capacity to the one created above.
//
// The network topology should now look like: Carol -> Alice -> Bob
carol, err := net.NewNode(nil)
if err != nil {
t.Fatalf("unable to create new nodes: %v", err)
}
if err := net.ConnectNodes(ctxb, carol, net.Alice); err != nil {
t.Fatalf("unable to connect carol to alice: %v", err)
}
err = net.SendCoins(ctxb, btcutil.SatoshiPerBitcoin, carol)
if err != nil {
t.Fatalf("unable to send coins to carol: %v", err)
}
ctxt, _ = context.WithTimeout(ctxb, timeout)
chanPointCarol := openChannelAndAssert(t, net, ctxt, carol,
net.Alice, chanAmt)
carolChanTXID, err := wire.NewShaHash(chanPointCarol.FundingTxid)
if err != nil {
t.Fatalf("unable to create sha hash: %v", err)
}
carolFundPoint := wire.OutPoint{
Hash: *carolChanTXID,
Index: chanPointCarol.OutputIndex,
}
// Create 5 invoices for Bob, which expect a payment from Carol for 1k
// satoshis with a different preimage each time.
const numPayments = 5
const paymentAmt = 1000
rHashes := make([][]byte, numPayments)
for i := 0; i < numPayments; i++ {
preimage := bytes.Repeat([]byte{byte(i)}, 32)
invoice := &lnrpc.Invoice{
Memo: "testing",
RPreimage: preimage,
Value: paymentAmt,
}
resp, err := net.Bob.AddInvoice(ctxb, invoice)
if err != nil {
t.Fatalf("unable to add invoice: %v", err)
}
rHashes[i] = resp.RHash
}
// Carol's routing table should show a path from Carol -> Alice -> Bob,
// with the two channels above recognized as the only links within the
// network.
time.Sleep(time.Second)
req := &lnrpc.ShowRoutingTableRequest{}
routingResp, err := carol.ShowRoutingTable(ctxb, req)
if err != nil {
t.Fatalf("unable to query for carol's routing table: %v", err)
}
if len(routingResp.Channels) != 2 {
t.Fatalf("only two channels should be seen as active in the "+
"network, instead %v are", len(routingResp.Channels))
}
for _, link := range routingResp.Channels {
switch {
case link.Outpoint == aliceFundPoint.String():
switch {
case link.Id1 == net.Alice.PubKeyStr &&
link.Id2 == net.Bob.PubKeyStr:
continue
case link.Id1 == net.Bob.PubKeyStr &&
link.Id2 == net.Alice.PubKeyStr:
continue
default:
t.Fatalf("unkown link within routing "+
"table: %v", spew.Sdump(link))
}
case link.Outpoint == carolFundPoint.String():
switch {
case link.Id1 == net.Alice.PubKeyStr &&
link.Id2 == carol.PubKeyStr:
continue
case link.Id1 == carol.PubKeyStr &&
link.Id2 == net.Alice.PubKeyStr:
continue
default:
t.Fatalf("unkown link within routing "+
"table: %v", spew.Sdump(link))
}
default:
t.Fatalf("unkown channel %v found in routing table, "+
"only %v and %v should exist", link.Outpoint,
aliceFundPoint, carolFundPoint)
}
}
// Using Carol as the source, pay to the 5 invoices from Bob created above.
carolPayStream, err := carol.SendPayment(ctxb)
if err != nil {
t.Fatalf("unable to create payment stream for carol: %v", err)
}
// Concurrently pay off all 5 of Bob's invoices. Each of the goroutines
// will unblock on the recv once the HTLC it sent has been fully
// settled.
var wg sync.WaitGroup
for _, rHash := range rHashes {
sendReq := &lnrpc.SendRequest{
PaymentHash: rHash,
Dest: net.Bob.PubKey[:],
Amt: paymentAmt,
}
wg.Add(1)
go func() {
if err := carolPayStream.Send(sendReq); err != nil {
t.Fatalf("unable to send payment: %v", err)
}
if _, err := carolPayStream.Recv(); err != nil {
t.Fatalf("unable to recv pay resp: %v", err)
}
wg.Done()
}()
}
finClear := make(chan struct{})
go func() {
wg.Wait()
close(finClear)
}()
select {
case <-time.After(time.Second * 10):
t.Fatalf("HTLC's not cleared after 10 seconds")
case <-finClear:
}
assertAsymmetricBalance := func(node *lightningNode,
chanPoint wire.OutPoint, localBalance,
remoteBalance int64) {
channelName := ""
switch chanPoint {
case carolFundPoint:
channelName = "Carol(local) => Alice(remote)"
case aliceFundPoint:
channelName = "Alice(local) => Bob(remote)"
}
checkBalance := func() error {
listReq := &lnrpc.ListChannelsRequest{}
resp, err := node.ListChannels(ctxb, listReq)
if err != nil {
return fmt.Errorf("unable to for node's "+
"channels: %v", err)
}
for _, channel := range resp.Channels {
if channel.ChannelPoint != chanPoint.String() {
continue
}
if channel.LocalBalance != localBalance {
return fmt.Errorf("%v: incorrect local "+
"balances: %v != %v", channelName,
channel.LocalBalance, localBalance)
}
if channel.RemoteBalance != remoteBalance {
return fmt.Errorf("%v: incorrect remote "+
"balances: %v != %v", channelName,
channel.RemoteBalance, remoteBalance)
}
return nil
}
return fmt.Errorf("channel not found")
}
// As far as HTLC inclusion in commitment transaction might be
// postponed we will try to check the balance couple of
// times, and then if after some period of time we receive wrong
// balance return the error.
// TODO(roasbeef): remove sleep after invoice notification hooks
// are in place
var timeover uint32
go func() {
<-time.After(time.Second * 20)
atomic.StoreUint32(&timeover, 1)
}()
for {
isTimeover := atomic.LoadUint32(&timeover) == 1
if err := checkBalance(); err != nil {
if isTimeover {
t.Fatalf("Check balance failed: %v", err)
}
} else {
break
}
}
}
// At this point all the channels within our proto network should be
// shifted by 5k satoshis in the direction of Bob, the sink within the
// payment flow generated above. The order of asserts corresponds to
// increasing of time is needed to embed the HTLC in commitment
// transaction, in channel Carol->Alice->Bob, order is Bob,Alice,Carol.
const sourceBal = int64(95000)
const sinkBal = int64(5000)
assertAsymmetricBalance(net.Bob, aliceFundPoint, sinkBal, sourceBal)
assertAsymmetricBalance(net.Alice, aliceFundPoint, sourceBal, sinkBal)
assertAsymmetricBalance(net.Alice, carolFundPoint, sinkBal, sourceBal)
assertAsymmetricBalance(carol, carolFundPoint, sourceBal, sinkBal)
ctxt, _ = context.WithTimeout(ctxb, timeout)
closeChannelAndAssert(t, net, ctxt, net.Alice, chanPointAlice)
ctxt, _ = context.WithTimeout(ctxb, timeout)
closeChannelAndAssert(t, net, ctxt, carol, chanPointCarol)
}