func InitializeSupernodeNetwork(
ctx context.Context,
numServers, numClients int,
conf testutil.LatencyConfig) ([]*core.IpfsNode, []*core.IpfsNode, error) {
// create network
mn := mocknet.New(ctx)
mn.SetLinkDefaults(mocknet.LinkOptions{
Latency: conf.NetworkLatency,
Bandwidth: math.MaxInt32,
})
routingDatastore := ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore()))
var servers []*core.IpfsNode
for i := 0; i < numServers; i++ {
bootstrap, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
Routing: corerouting.SupernodeServer(routingDatastore),
})
if err != nil {
return nil, nil, err
}
servers = append(servers, bootstrap)
}
var bootstrapInfos []peer.PeerInfo
for _, n := range servers {
info := n.Peerstore.PeerInfo(n.PeerHost.ID())
bootstrapInfos = append(bootstrapInfos, info)
}
var clients []*core.IpfsNode
for i := 0; i < numClients; i++ {
n, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
Routing: corerouting.SupernodeClient(bootstrapInfos...),
})
if err != nil {
return nil, nil, err
}
clients = append(clients, n)
}
mn.LinkAll()
bcfg := core.BootstrapConfigWithPeers(bootstrapInfos)
for _, n := range clients {
if err := n.Bootstrap(bcfg); err != nil {
return nil, nil, err
}
}
return servers, clients, nil
}
func (i *cmdInvocation) constructNodeFunc(ctx context.Context) func() (*core.IpfsNode, error) {
return func() (*core.IpfsNode, error) {
if i.req == nil {
return nil, errors.New("constructing node without a request")
}
cmdctx := i.req.InvocContext()
if cmdctx == nil {
return nil, errors.New("constructing node without a request context")
}
r, err := fsrepo.Open(i.req.InvocContext().ConfigRoot)
if err != nil { // repo is owned by the node
return nil, err
}
// ok everything is good. set it on the invocation (for ownership)
// and return it.
n, err := core.NewNode(ctx, &core.BuildCfg{
Online: cmdctx.Online,
Repo: r,
})
if err != nil {
return nil, err
}
i.node = n
return i.node, nil
}
}
func addDefaultAssets(out io.Writer, repoRoot string) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
r, err := fsrepo.Open(repoRoot)
if err != nil { // NB: repo is owned by the node
return err
}
nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r})
if err != nil {
return err
}
defer nd.Close()
dkey, err := assets.SeedInitDocs(nd)
if err != nil {
return fmt.Errorf("init: seeding init docs failed: %s", err)
}
log.Debugf("init: seeded init docs %s", dkey)
if _, err = fmt.Fprintf(out, "to get started, enter:\n"); err != nil {
return err
}
_, err = fmt.Fprintf(out, "\n\tipfs cat /ipfs/%s/readme\n\n", dkey)
return err
}
func MockCmdsCtx() (commands.Context, error) {
// Generate Identity
ident, err := testutil.RandIdentity()
if err != nil {
return commands.Context{}, err
}
p := ident.ID()
conf := config.Config{
Identity: config.Identity{
PeerID: p.String(),
},
}
r := &repo.Mock{
D: ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())),
C: conf,
}
node, err := core.NewNode(context.Background(), &core.BuildCfg{
Repo: r,
})
return commands.Context{
Online: true,
ConfigRoot: "/tmp/.mockipfsconfig",
LoadConfig: func(path string) (*config.Config, error) {
return &conf, nil
},
ConstructNode: func() (*core.IpfsNode, error) {
return node, nil
},
}, nil
}
// NewMockNode constructs an IpfsNode for use in tests.
func NewMockNode() (*core.IpfsNode, error) {
ctx := context.Background()
// effectively offline, only peer in its network
return core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: MockHostOption(mocknet.New(ctx)),
})
}
func runFileCattingWorker(ctx context.Context, n *core.IpfsNode) error {
conf, err := config.Init(ioutil.Discard, *nBitsForKeypair)
if err != nil {
return err
}
r := &repo.Mock{
D: ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())),
C: *conf,
}
dummy, err := core.NewNode(ctx, &core.BuildCfg{
Repo: r,
})
if err != nil {
return err
}
go func() {
defer dummy.Close()
var i int64 = 1
for {
buf := new(bytes.Buffer)
if err := random.WritePseudoRandomBytes(sizeOfIthFile(i), buf, *seed); err != nil {
log.Fatal(err)
}
// add to a dummy node to discover the key
k, err := coreunix.Add(dummy, bytes.NewReader(buf.Bytes()))
if err != nil {
log.Fatal(err)
}
e := elog.EventBegin(ctx, "cat", logging.LoggableF(func() map[string]interface{} {
return map[string]interface{}{
"key": k,
"localPeer": n.Identity,
}
}))
if r, err := coreunix.Cat(n, k); err != nil {
e.Done()
log.Printf("failed to cat file. seed: %d #%d key: %s err: %s", *seed, i, k, err)
} else {
log.Println("found file", "seed", *seed, "#", i, "key", k, "size", unit.Information(sizeOfIthFile(i)))
io.Copy(ioutil.Discard, r)
e.Done()
log.Println("catted file", "seed", *seed, "#", i, "key", k, "size", unit.Information(sizeOfIthFile(i)))
i++
}
time.Sleep(time.Second)
}
}()
return nil
}
func newNodeWithMockNamesys(ns mockNamesys) (*core.IpfsNode, error) {
c := config.Config{
Identity: config.Identity{
PeerID: "Qmfoo", // required by offline node
},
}
r := &repo.Mock{
C: c,
D: testutil.ThreadSafeCloserMapDatastore(),
}
n, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
if err != nil {
return nil, err
}
n.Namesys = ns
return n, nil
}
func initializeIpnsKeyspace(repoRoot string) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
r, err := fsrepo.Open(repoRoot)
if err != nil { // NB: repo is owned by the node
return err
}
nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r})
if err != nil {
return err
}
defer nd.Close()
err = nd.SetupOfflineRouting()
if err != nil {
return err
}
return namesys.InitializeKeyspace(ctx, nd.DAG, nd.Namesys, nd.Pinning, nd.PrivateKey)
}
func TestAddRecursive(t *testing.T) {
here, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
r := &repo.Mock{
C: config.Config{
Identity: config.Identity{
PeerID: "Qmfoo", // required by offline node
},
},
D: testutil.ThreadSafeCloserMapDatastore(),
}
node, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
if err != nil {
t.Fatal(err)
}
if k, err := AddR(node, path.Join(here, "test_data")); err != nil {
t.Fatal(err)
} else if k != "QmWCCga8AbTyfAQ7pTnGT6JgmRMAB3Qp8ZmTEFi5q5o8jC" {
t.Fatal("keys do not match")
}
}
func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) {
maybeSkipFuseTests(t)
var err error
if node == nil {
node, err = core.NewNode(context.Background(), nil)
if err != nil {
t.Fatal(err)
}
err = node.LoadPrivateKey()
if err != nil {
t.Fatal(err)
}
node.Routing = offroute.NewOfflineRouter(node.Repo.Datastore(), node.PrivateKey)
node.Namesys = namesys.NewNameSystem(node.Routing)
ipnsfs, err := nsfs.NewFilesystem(context.Background(), node.DAG, node.Namesys, node.Pinning, node.PrivateKey)
if err != nil {
t.Fatal(err)
}
node.IpnsFs = ipnsfs
}
fs, err := NewFileSystem(node, node.PrivateKey, "", "")
if err != nil {
t.Fatal(err)
}
mnt, err := fstest.MountedT(t, fs)
if err != nil {
t.Fatal(err)
}
return node, mnt
}
func TestBitswapWithoutRouting(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
const numPeers = 4
// create network
mn := mocknet.New(ctx)
var nodes []*core.IpfsNode
for i := 0; i < numPeers; i++ {
n, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: coremock.MockHostOption(mn),
Routing: core.NilRouterOption, // no routing
})
if err != nil {
t.Fatal(err)
}
defer n.Close()
nodes = append(nodes, n)
}
mn.LinkAll()
// connect them
for _, n1 := range nodes {
for _, n2 := range nodes {
if n1 == n2 {
continue
}
log.Debug("connecting to other hosts")
p2 := n2.PeerHost.Peerstore().PeerInfo(n2.PeerHost.ID())
if err := n1.PeerHost.Connect(ctx, p2); err != nil {
t.Fatal(err)
}
}
}
// add blocks to each before
log.Debug("adding block.")
block0 := blocks.NewBlock([]byte("block0"))
block1 := blocks.NewBlock([]byte("block1"))
// put 1 before
if err := nodes[0].Blockstore.Put(block0); err != nil {
t.Fatal(err)
}
// get it out.
for i, n := range nodes {
// skip first because block not in its exchange. will hang.
if i == 0 {
continue
}
log.Debugf("%d %s get block.", i, n.Identity)
b, err := n.Blocks.GetBlock(ctx, block0.Key())
if err != nil {
t.Error(err)
} else if !bytes.Equal(b.Data, block0.Data) {
t.Error("byte comparison fail")
} else {
log.Debug("got block: %s", b.Key())
}
}
// put 1 after
if err := nodes[1].Blockstore.Put(block1); err != nil {
t.Fatal(err)
}
// get it out.
for _, n := range nodes {
b, err := n.Blocks.GetBlock(ctx, block1.Key())
if err != nil {
t.Error(err)
} else if !bytes.Equal(b.Data, block1.Data) {
t.Error("byte comparison fail")
} else {
log.Debug("got block: %s", b.Key())
}
}
}
func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
const numPeers = 3
// create network
mn := mocknet.New(ctx)
mn.SetLinkDefaults(mocknet.LinkOptions{
Latency: conf.NetworkLatency,
// TODO add to conf. This is tricky because we want 0 values to be functional.
Bandwidth: math.MaxInt32,
})
bootstrap, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer bootstrap.Close()
adder, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer adder.Close()
catter, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer catter.Close()
mn.LinkAll()
bis := bootstrap.Peerstore.PeerInfo(bootstrap.PeerHost.ID())
bcfg := core.BootstrapConfigWithPeers([]peer.PeerInfo{bis})
if err := adder.Bootstrap(bcfg); err != nil {
return err
}
if err := catter.Bootstrap(bcfg); err != nil {
return err
}
added, err := coreunix.Add(adder, bytes.NewReader(data))
if err != nil {
return err
}
readerCatted, err := coreunix.Cat(catter, added)
if err != nil {
return err
}
// verify
bufout := new(bytes.Buffer)
io.Copy(bufout, readerCatted)
if 0 != bytes.Compare(bufout.Bytes(), data) {
return errors.New("catted data does not match added data")
}
return nil
}
func run() error {
servers := config.DefaultSNRServers
fmt.Println("using gcr remotes:")
for _, p := range servers {
fmt.Println("\t", p)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cwd, err := os.Getwd()
if err != nil {
return err
}
repoPath := gopath.Join(cwd, config.DefaultPathName)
if err := ensureRepoInitialized(repoPath); err != nil {
}
repo, err := fsrepo.Open(repoPath)
if err != nil { // owned by node
return err
}
cfg, err := repo.Config()
if err != nil {
return err
}
cfg.Bootstrap = servers
if err := repo.SetConfig(cfg); err != nil {
return err
}
var addrs []ipfsaddr.IPFSAddr
for _, info := range servers {
addr, err := ipfsaddr.ParseString(info)
if err != nil {
return err
}
addrs = append(addrs, addr)
}
var infos []peer.PeerInfo
for _, addr := range addrs {
infos = append(infos, peer.PeerInfo{
ID: addr.ID(),
Addrs: []ma.Multiaddr{addr.Transport()},
})
}
node, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Repo: repo,
Routing: corerouting.SupernodeClient(infos...),
})
if err != nil {
return err
}
defer node.Close()
opts := []corehttp.ServeOption{
corehttp.CommandsOption(cmdCtx(node, repoPath)),
corehttp.GatewayOption(false),
}
if *cat {
if err := runFileCattingWorker(ctx, node); err != nil {
return err
}
} else {
if err := runFileAddingWorker(node); err != nil {
return err
}
}
return corehttp.ListenAndServe(node, cfg.Addresses.API, opts...)
}
func benchCat(b *testing.B, data []byte, conf testutil.LatencyConfig) error {
b.StopTimer()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create network
mn := mocknet.New(ctx)
mn.SetLinkDefaults(mocknet.LinkOptions{
Latency: conf.NetworkLatency,
// TODO add to conf. This is tricky because we want 0 values to be functional.
Bandwidth: math.MaxInt32,
})
adder, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer adder.Close()
catter, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer catter.Close()
err = mn.LinkAll()
if err != nil {
return err
}
bs1 := []peer.PeerInfo{adder.Peerstore.PeerInfo(adder.Identity)}
bs2 := []peer.PeerInfo{catter.Peerstore.PeerInfo(catter.Identity)}
if err := catter.Bootstrap(core.BootstrapConfigWithPeers(bs1)); err != nil {
return err
}
if err := adder.Bootstrap(core.BootstrapConfigWithPeers(bs2)); err != nil {
return err
}
added, err := coreunix.Add(adder, bytes.NewReader(data))
if err != nil {
return err
}
b.StartTimer()
readerCatted, err := coreunix.Cat(catter, added)
if err != nil {
return err
}
// verify
bufout := new(bytes.Buffer)
io.Copy(bufout, readerCatted)
if 0 != bytes.Compare(bufout.Bytes(), data) {
return errors.New("catted data does not match added data")
}
return nil
}
func daemonFunc(req cmds.Request, res cmds.Response) {
// let the user know we're going.
fmt.Printf("Initializing daemon...\n")
ctx := req.InvocContext()
go func() {
select {
case <-req.Context().Done():
fmt.Println("Received interrupt signal, shutting down...")
}
}()
// check transport encryption flag.
unencrypted, _, _ := req.Option(unencryptTransportKwd).Bool()
if unencrypted {
log.Warningf(`Running with --%s: All connections are UNENCRYPTED.
You will not be able to connect to regular encrypted networks.`, unencryptTransportKwd)
conn.EncryptConnections = false
}
// first, whether user has provided the initialization flag. we may be
// running in an uninitialized state.
initialize, _, err := req.Option(initOptionKwd).Bool()
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
if initialize {
// now, FileExists is our best method of detecting whether IPFS is
// configured. Consider moving this into a config helper method
// `IsInitialized` where the quality of the signal can be improved over
// time, and many call-sites can benefit.
if !util.FileExists(req.InvocContext().ConfigRoot) {
err := initWithDefaults(os.Stdout, req.InvocContext().ConfigRoot)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
}
}
// acquire the repo lock _before_ constructing a node. we need to make
// sure we are permitted to access the resources (datastore, etc.)
repo, err := fsrepo.Open(req.InvocContext().ConfigRoot)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
cfg, err := ctx.GetConfig()
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
// Start assembling node config
ncfg := &core.BuildCfg{
Online: true,
Repo: repo,
}
routingOption, _, err := req.Option(routingOptionKwd).String()
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
if routingOption == routingOptionSupernodeKwd {
servers, err := cfg.SupernodeRouting.ServerIPFSAddrs()
if err != nil {
res.SetError(err, cmds.ErrNormal)
repo.Close() // because ownership hasn't been transferred to the node
return
}
var infos []peer.PeerInfo
for _, addr := range servers {
infos = append(infos, peer.PeerInfo{
ID: addr.ID(),
Addrs: []ma.Multiaddr{addr.Transport()},
})
}
ncfg.Routing = corerouting.SupernodeClient(infos...)
}
node, err := core.NewNode(req.Context(), ncfg)
if err != nil {
log.Error("error from node construction: ", err)
res.SetError(err, cmds.ErrNormal)
return
}
printSwarmAddrs(node)
defer func() {
// We wait for the node to close first, as the node has children
// that it will wait for before closing, such as the API server.
node.Close()
select {
case <-req.Context().Done():
log.Info("Gracefully shut down daemon")
default:
}
}()
req.InvocContext().ConstructNode = func() (*core.IpfsNode, error) {
return node, nil
}
// construct api endpoint - every time
err, apiErrc := serveHTTPApi(req)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
// construct http gateway - if it is set in the config
var gwErrc <-chan error
if len(cfg.Addresses.Gateway) > 0 {
var err error
err, gwErrc = serveHTTPGateway(req)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
}
// construct fuse mountpoints - if the user provided the --mount flag
mount, _, err := req.Option(mountKwd).Bool()
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
if mount {
if err := mountFuse(req); err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
}
fmt.Printf("Daemon is ready\n")
// collect long-running errors and block for shutdown
// TODO(cryptix): our fuse currently doesnt follow this pattern for graceful shutdown
for err := range merge(apiErrc, gwErrc) {
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
}
}
func run(ipfsPath, watchPath string) error {
proc := process.WithParent(process.Background())
log.Printf("running IPFSWatch on '%s' using repo at '%s'...", watchPath, ipfsPath)
ipfsPath, err := homedir.Expand(ipfsPath)
if err != nil {
return err
}
watcher, err := fsnotify.NewWatcher()
if err != nil {
return err
}
defer watcher.Close()
if err := addTree(watcher, watchPath); err != nil {
return err
}
r, err := fsrepo.Open(ipfsPath)
if err != nil {
// TODO handle case: daemon running
// TODO handle case: repo doesn't exist or isn't initialized
return err
}
node, err := core.NewNode(context.Background(), &core.BuildCfg{
Online: true,
Repo: r,
})
if err != nil {
return err
}
defer node.Close()
if *http {
addr := "/ip4/127.0.0.1/tcp/5001"
var opts = []corehttp.ServeOption{
corehttp.GatewayOption(true),
corehttp.WebUIOption,
corehttp.CommandsOption(cmdCtx(node, ipfsPath)),
}
proc.Go(func(p process.Process) {
if err := corehttp.ListenAndServe(node, addr, opts...); err != nil {
return
}
})
}
interrupts := make(chan os.Signal)
signal.Notify(interrupts, os.Interrupt, os.Kill)
for {
select {
case <-interrupts:
return nil
case e := <-watcher.Events:
log.Printf("received event: %s", e)
isDir, err := IsDirectory(e.Name)
if err != nil {
continue
}
switch e.Op {
case fsnotify.Remove:
if isDir {
if err := watcher.Remove(e.Name); err != nil {
return err
}
}
default:
// all events except for Remove result in an IPFS.Add, but only
// directory creation triggers a new watch
switch e.Op {
case fsnotify.Create:
if isDir {
addTree(watcher, e.Name)
}
}
proc.Go(func(p process.Process) {
file, err := os.Open(e.Name)
if err != nil {
log.Println(err)
}
defer file.Close()
k, err := coreunix.Add(node, file)
if err != nil {
log.Println(err)
}
log.Printf("added %s... key: %s", e.Name, k)
})
}
case err := <-watcher.Errors:
log.Println(err)
}
}
return nil
}
res.SetError(err, cmds.ErrNormal)
return
}
progress, _, _ := req.Option(progressOptionName).Bool()
trickle, _, _ := req.Option(trickleOptionName).Bool()
wrap, _, _ := req.Option(wrapOptionName).Bool()
hash, _, _ := req.Option(onlyHashOptionName).Bool()
hidden, _, _ := req.Option(hiddenOptionName).Bool()
chunker, _, _ := req.Option(chunkerOptionName).String()
e := dagutils.NewDagEditor(NewMemoryDagService(), newDirNode())
if hash {
nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
//TODO: need this to be true or all files
// hashed will be stored in memory!
NilRepo: true,
})
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
n = nilnode
}
outChan := make(chan interface{}, 8)
res.SetOutput((<-chan interface{})(outChan))
fileAdder := adder{
ctx: req.Context(),
node: n,