// TestIntegrationManyHeaders checks that requesting a full set of headers in a
// row results in all unique headers, and that all of them can be reassembled
// into valid blocks.
func TestIntegrationManyHeaders(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
mt, err := createMinerTester("TestIntegrationManyHeaders")
if err != nil {
t.Fatal(err)
}
// Create a suite of headers for imaginary parallel mining.
solvedHeaders := make([]types.BlockHeader, HeaderMemory/BlockMemory*2)
for i := range solvedHeaders {
header, target, err := mt.miner.HeaderForWork()
if err != nil {
t.Fatal(err)
}
solvedHeaders[i] = solveHeader(header, target)
}
// Submit the headers randomly and make sure they are all considered valid.
selectionOrder, err := crypto.Perm(len(solvedHeaders))
if err != nil {
t.Fatal(err)
}
for _, selection := range selectionOrder {
err = mt.miner.SubmitHeader(solvedHeaders[selection])
if err != nil && err != modules.ErrNonExtendingBlock {
t.Error(err)
}
}
}
// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
defer hdb.threadGroup.Done()
for {
// Determine who to scan. At most 'maxActiveHosts' will be scanned,
// starting with the active hosts followed by a random selection of the
// inactive hosts.
func() {
hdb.mu.Lock()
defer hdb.mu.Unlock()
// Scan all active hosts.
for _, host := range hdb.activeHosts {
hdb.scanHostEntry(host.hostEntry)
}
// Assemble all of the inactive hosts into a single array.
var entries []*hostEntry
for _, entry := range hdb.allHosts {
_, exists := hdb.activeHosts[entry.NetAddress]
if !exists {
entries = append(entries, entry)
}
}
// Generate a random ordering of up to inactiveHostCheckupQuantity
// hosts.
hostOrder, err := crypto.Perm(len(entries))
if err != nil {
hdb.log.Println("ERR: could not generate random permutation:", err)
}
// Scan each host.
for i := 0; i < len(hostOrder) && i < inactiveHostCheckupQuantity; i++ {
hdb.scanHostEntry(entries[hostOrder[i]])
}
}()
// Sleep for a random amount of time before doing another round of
// scanning. The minimums and maximums keep the scan time reasonable,
// while the randomness prevents the scanning from always happening at
// the same time of day or week.
maxBig := big.NewInt(int64(maxScanSleep))
minBig := big.NewInt(int64(minScanSleep))
randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
if err != nil {
build.Critical(err)
// If there's an error, sleep for the default amount of time.
defaultBig := big.NewInt(int64(defaultScanSleep))
randSleep = defaultBig.Sub(defaultBig, minBig)
}
select {
// awaken and exit if hostdb is closing
case <-hdb.closeChan:
return
case <-time.After(time.Duration(randSleep.Int64()) + minScanSleep):
}
}
}
// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
for {
// Determine who to scan. At most 'MaxActiveHosts' will be scanned,
// starting with the active hosts followed by a random selection of the
// inactive hosts.
func() {
hdb.mu.Lock()
defer hdb.mu.Unlock()
// Scan all active hosts.
for _, host := range hdb.activeHosts {
hdb.scanHostEntry(host.hostEntry)
}
// Assemble all of the inactive hosts into a single array.
var entries []*hostEntry
for _, entry := range hdb.allHosts {
entry2, exists := hdb.activeHosts[entry.NetAddress]
if !exists {
entries = append(entries, entry)
} else if entry2.hostEntry != entry {
build.Critical("allHosts + activeHosts mismatch!")
}
}
// Generate a random ordering of up to InactiveHostCheckupQuantity
// hosts.
n := InactiveHostCheckupQuantity
if n > len(entries) {
n = len(entries)
}
hostOrder, err := crypto.Perm(n)
if err != nil {
hdb.log.Println("ERR: could not generate random permutation:", err)
}
// Scan each host.
for _, randIndex := range hostOrder {
hdb.scanHostEntry(entries[randIndex])
}
}()
// Sleep for a random amount of time before doing another round of
// scanning. The minimums and maximums keep the scan time reasonable,
// while the randomness prevents the scanning from always happening at
// the same time of day or week.
maxBig := big.NewInt(int64(MaxScanSleep))
minBig := big.NewInt(int64(MinScanSleep))
randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
if err != nil {
build.Critical(err)
// If there's an error, sleep for the default amount of time.
defaultBig := big.NewInt(int64(DefaultScanSleep))
randSleep = defaultBig.Sub(defaultBig, minBig)
}
hdb.sleeper.Sleep(time.Duration(randSleep.Int64()) + MinScanSleep) // this means the MaxScanSleep is actual Max+Min.
}
}
// shareNodes is the receiving end of the ShareNodes RPC. It writes up to 10
// randomly selected nodes to the caller.
func (g *Gateway) shareNodes(conn modules.PeerConn) error {
conn.SetDeadline(time.Now().Add(connStdDeadline))
// Assemble a list of nodes to send to the peer.
var nodes []modules.NetAddress
func() {
g.mu.RLock()
defer g.mu.RUnlock()
// Create a random permutation of nodes from the gateway to iterate
// through.
gnodes := make([]modules.NetAddress, 0, len(g.nodes))
for node := range g.nodes {
gnodes = append(gnodes, node)
}
perm, err := crypto.Perm(len(g.nodes))
if err != nil {
g.log.Severe("Unable to get random permutation for sharing nodes")
}
// Iterate through the random permutation of nodes and select the
// desirable ones.
remoteNA := modules.NetAddress(conn.RemoteAddr().String())
for _, i := range perm {
// Don't share local peers with remote peers. That means that if 'node'
// is loopback, it will only be shared if the remote peer is also
// loopback. And if 'node' is private, it will only be shared if the
// remote peer is either the loopback or is also private.
node := gnodes[i]
if node.IsLoopback() && !remoteNA.IsLoopback() {
continue
}
if node.IsLocal() && !remoteNA.IsLocal() {
continue
}
nodes = append(nodes, node)
if uint64(len(nodes)) == maxSharedNodes {
break
}
}
}()
return encoding.WriteObject(conn, nodes)
}
// run performs the actual download. It spawns one worker per host, and
// instructs them to sequentially download chunks. It then writes the
// recovered chunks to w.
func (d *download) run(w io.Writer) error {
var received uint64
for i := uint64(0); received < d.fileSize; i++ {
// load pieces into chunk
chunk := make([][]byte, d.erasureCode.NumPieces())
left := d.erasureCode.MinPieces()
// pick hosts at random
chunkOrder, err := crypto.Perm(len(chunk))
if err != nil {
return err
}
for _, j := range chunkOrder {
chunk[j] = d.getPiece(i, uint64(j))
if chunk[j] != nil {
left--
} else {
}
if left == 0 {
break
}
}
if left != 0 {
return errInsufficientPieces
}
// Write pieces to w. We always write chunkSize bytes unless this is
// the last chunk; in that case, we write the remainder.
n := d.chunkSize
if n > d.fileSize-received {
n = d.fileSize - received
}
err = d.erasureCode.Recover(chunk, uint64(n), w)
if err != nil {
return err
}
received += n
atomic.AddUint64(&d.received, n)
}
return nil
}
// startDaemonCmd uses the config parameters to start siad.
func startDaemon(config Config) (err error) {
// Prompt user for API password.
var password string
if config.Siad.AuthenticateAPI {
password, err = speakeasy.Ask("Enter API password: "******"" {
return errors.New("password cannot be blank")
}
passwordConfirm, err := speakeasy.Ask("Confirm API password: "******"passwords don't match")
}
}
// Print a startup message.
fmt.Println("Loading...")
loadStart := time.Now()
// Create all of the modules.
i := 0
var g modules.Gateway
if strings.Contains(config.Siad.Modules, "g") {
i++
fmt.Printf("(%d/%d) Loading gateway...\n", i, len(config.Siad.Modules))
g, err = gateway.New(config.Siad.RPCaddr, filepath.Join(config.Siad.SiaDir, modules.GatewayDir))
if err != nil {
return err
}
}
var cs modules.ConsensusSet
if strings.Contains(config.Siad.Modules, "c") {
i++
fmt.Printf("(%d/%d) Loading consensus...\n", i, len(config.Siad.Modules))
cs, err = consensus.New(g, filepath.Join(config.Siad.SiaDir, modules.ConsensusDir))
if err != nil {
return err
}
}
var e modules.Explorer
if strings.Contains(config.Siad.Modules, "e") {
i++
fmt.Printf("(%d/%d) Loading explorer...\n", i, len(config.Siad.Modules))
e, err = explorer.New(cs, filepath.Join(config.Siad.SiaDir, modules.ExplorerDir))
if err != nil {
return err
}
}
var tpool modules.TransactionPool
if strings.Contains(config.Siad.Modules, "t") {
i++
fmt.Printf("(%d/%d) Loading transaction pool...\n", i, len(config.Siad.Modules))
tpool, err = transactionpool.New(cs, g, filepath.Join(config.Siad.SiaDir, modules.TransactionPoolDir))
if err != nil {
return err
}
}
var w modules.Wallet
if strings.Contains(config.Siad.Modules, "w") {
i++
fmt.Printf("(%d/%d) Loading wallet...\n", i, len(config.Siad.Modules))
w, err = wallet.New(cs, tpool, filepath.Join(config.Siad.SiaDir, modules.WalletDir))
if err != nil {
return err
}
}
var m modules.Miner
if strings.Contains(config.Siad.Modules, "m") {
i++
fmt.Printf("(%d/%d) Loading miner...\n", i, len(config.Siad.Modules))
m, err = miner.New(cs, tpool, w, filepath.Join(config.Siad.SiaDir, modules.MinerDir))
if err != nil {
return err
}
}
var h modules.Host
if strings.Contains(config.Siad.Modules, "h") {
i++
fmt.Printf("(%d/%d) Loading host...\n", i, len(config.Siad.Modules))
h, err = host.New(cs, tpool, w, config.Siad.HostAddr, filepath.Join(config.Siad.SiaDir, modules.HostDir))
if err != nil {
return err
}
}
var r modules.Renter
if strings.Contains(config.Siad.Modules, "r") {
i++
fmt.Printf("(%d/%d) Loading renter...\n", i, len(config.Siad.Modules))
r, err = renter.New(cs, w, tpool, filepath.Join(config.Siad.SiaDir, modules.RenterDir))
if err != nil {
return err
}
}
srv, err := api.NewServer(
config.Siad.APIaddr,
config.Siad.RequiredUserAgent,
password,
cs,
e,
g,
h,
m,
r,
tpool,
w,
)
if err != nil {
return err
}
// Bootstrap to the network.
if !config.Siad.NoBootstrap && g != nil {
// connect to 3 random bootstrap nodes
perm, err := crypto.Perm(len(modules.BootstrapPeers))
if err != nil {
return err
}
for _, i := range perm[:3] {
go g.Connect(modules.BootstrapPeers[i])
}
}
// Print a 'startup complete' message.
startupTime := time.Since(loadStart)
fmt.Println("Finished loading in", startupTime.Seconds(), "seconds")
// Start serving api requests.
err = srv.Serve()
if err != nil {
return err
}
return nil
}
// startDaemonCmd uses the config parameters to start siad.
func startDaemon(config Config) error {
// Print a startup message.
fmt.Println("Loading...")
loadStart := time.Now()
// Create all of the modules.
gateway, err := gateway.New(config.Siad.RPCaddr, filepath.Join(config.Siad.SiaDir, modules.GatewayDir))
if err != nil {
return err
}
cs, err := consensus.New(gateway, filepath.Join(config.Siad.SiaDir, modules.ConsensusDir))
if err != nil {
return err
}
var e *explorer.Explorer
if config.Siad.Explorer {
e, err = explorer.New(cs, filepath.Join(config.Siad.SiaDir, modules.ExplorerDir))
if err != nil {
return err
}
}
tpool, err := transactionpool.New(cs, gateway)
if err != nil {
return err
}
wallet, err := wallet.New(cs, tpool, filepath.Join(config.Siad.SiaDir, modules.WalletDir))
if err != nil {
return err
}
miner, err := miner.New(cs, tpool, wallet, filepath.Join(config.Siad.SiaDir, modules.MinerDir))
if err != nil {
return err
}
host, err := host.New(cs, tpool, wallet, config.Siad.HostAddr, filepath.Join(config.Siad.SiaDir, modules.HostDir))
if err != nil {
return err
}
renter, err := renter.New(cs, wallet, tpool, filepath.Join(config.Siad.SiaDir, modules.RenterDir))
if err != nil {
return err
}
srv, err := api.NewServer(
config.Siad.APIaddr,
config.Siad.RequiredUserAgent,
cs,
e,
gateway,
host,
miner,
renter,
tpool,
wallet,
)
if err != nil {
return err
}
// Bootstrap to the network.
if !config.Siad.NoBootstrap {
// connect to 3 random bootstrap nodes
perm, err := crypto.Perm(len(modules.BootstrapPeers))
if err != nil {
return err
}
for _, i := range perm[:3] {
go gateway.Connect(modules.BootstrapPeers[i])
}
}
// Print a 'startup complete' message.
startupTime := time.Since(loadStart)
fmt.Println("Finished loading in", startupTime.Seconds(), "seconds")
// Start serving api requests.
err = srv.Serve()
if err != nil {
return err
}
return nil
}
// startDaemonCmd uses the config parameters to start siad.
func startDaemon() error {
// Establish multithreading.
runtime.GOMAXPROCS(runtime.NumCPU())
// Print a startup message.
//
// TODO: This message can be removed once the api starts up in under 1/2
// second.
fmt.Println("Loading...")
loadStart := time.Now()
// Create all of the modules.
gateway, err := gateway.New(config.Siad.RPCaddr, filepath.Join(config.Siad.SiaDir, modules.GatewayDir))
if err != nil {
return err
}
cs, err := consensus.New(gateway, filepath.Join(config.Siad.SiaDir, modules.ConsensusDir))
if err != nil {
return err
}
tpool, err := transactionpool.New(cs, gateway)
if err != nil {
return err
}
wallet, err := wallet.New(cs, tpool, filepath.Join(config.Siad.SiaDir, modules.WalletDir))
if err != nil {
return err
}
miner, err := miner.New(cs, tpool, wallet, filepath.Join(config.Siad.SiaDir, modules.MinerDir))
if err != nil {
return err
}
host, err := host.New(cs, tpool, wallet, config.Siad.HostAddr, filepath.Join(config.Siad.SiaDir, modules.HostDir))
if err != nil {
return err
}
renter, err := renter.New(cs, wallet, tpool, filepath.Join(config.Siad.SiaDir, modules.RenterDir))
if err != nil {
return err
}
srv, err := api.NewServer(config.Siad.APIaddr, cs, gateway, host, miner, renter, tpool, wallet, nil)
if err != nil {
return err
}
// Bootstrap to the network.
if !config.Siad.NoBootstrap {
// connect to 3 random bootstrap nodes
perm := crypto.Perm(len(modules.BootstrapPeers))
for _, i := range perm[:3] {
go gateway.Connect(modules.BootstrapPeers[i])
}
}
// Print a 'startup complete' message.
//
// TODO: This message can be removed once the api starts up in under 1/2
// second.
startupTime := time.Since(loadStart)
fmt.Println("Finished loading in", startupTime.Seconds(), "seconds")
// Start serving api requests.
err = srv.Serve()
if err != nil {
return err
}
return nil
}
// TestOverloadedBootstrap creates a bunch of gateways and connects all of them
// to the first gateway, the bootstrap gateway. More gateways will be created
// than is allowed by the bootstrap for the total number of connections. After
// waiting, all peers should eventually get to the full number of outbound
// peers.
func TestOverloadedBootstrap(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
t.Parallel()
// Create fullyConnectedThreshold*2 peers and connect them all to only the
// first node.
var gs []*Gateway
for i := 0; i < fullyConnectedThreshold*2; i++ {
gname := "TestOverloadedBootstrap" + strconv.Itoa(i)
gs = append(gs, newTestingGateway(gname, t))
// Connect this gateway to the first gateway.
if i == 0 {
continue
}
err := gs[i].Connect(gs[0].myAddr)
for j := 0; j < 100 && err != nil; j++ {
time.Sleep(time.Millisecond * 250)
err = gs[i].Connect(gs[0].myAddr)
}
if err != nil {
panic(err)
}
}
// Spin until all gateways have a complete number of outbound peers.
success := false
for i := 0; i < 100; i++ {
success = true
for _, g := range gs {
outboundPeers := 0
g.mu.RLock()
for _, p := range g.peers {
if !p.Inbound {
outboundPeers++
}
}
g.mu.RUnlock()
if outboundPeers < wellConnectedThreshold {
success = false
break
}
}
if !success {
time.Sleep(time.Second)
}
}
if !success {
for i, g := range gs {
outboundPeers := 0
g.mu.RLock()
for _, p := range g.peers {
if !p.Inbound {
outboundPeers++
}
}
g.mu.RUnlock()
t.Log("Gateway", i, ":", outboundPeers)
}
t.Fatal("after 100 seconds not all gateways able to become well connected")
}
// Randomly close many of the peers. For many peers, this should put them
// below the well connected threshold, but there are still enough nodes on
// the network that no partitions should occur.
var newGS []*Gateway
perm, err := crypto.Perm(len(gs))
if err != nil {
t.Fatal(err)
}
// Reorder the gateways.
for _, i := range perm {
newGS = append(newGS, gs[i])
}
cutSize := len(newGS) / 4
// Close the first many of the now-randomly-sorted gateways.
for _, g := range newGS[:cutSize] {
err := g.Close()
if err != nil {
t.Fatal(err)
}
}
// Set 'gs' equal to the remaining gateways.
gs = newGS[cutSize:]
// Spin until all gateways have a complete number of outbound peers. The
// test can fail if there are network partitions, however not a huge
// magnitude of nodes are being removed, and they all started with 4
// connections. A partition is unlikely.
success = false
for i := 0; i < 100; i++ {
success = true
for _, g := range gs {
outboundPeers := 0
g.mu.RLock()
for _, p := range g.peers {
if !p.Inbound {
outboundPeers++
}
}
g.mu.RUnlock()
if outboundPeers < wellConnectedThreshold {
success = false
break
}
}
if !success {
time.Sleep(time.Second)
}
}
if !success {
t.Fatal("after 100 seconds not all gateways able to become well connected")
}
// Close all remaining gateways.
for _, g := range gs {
err = g.Close()
if err != nil {
t.Error(err)
}
}
}