func InitializeSupernodeNetwork( ctx context.Context, numServers, numClients int, conf testutil.LatencyConfig) ([]*core.IpfsNode, []*core.IpfsNode, error) { // create network mn := mocknet.New(ctx) mn.SetLinkDefaults(mocknet.LinkOptions{ Latency: conf.NetworkLatency, Bandwidth: math.MaxInt32, }) routingDatastore := ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())) var servers []*core.IpfsNode for i := 0; i < numServers; i++ { bootstrap, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), Routing: corerouting.SupernodeServer(routingDatastore), }) if err != nil { return nil, nil, err } servers = append(servers, bootstrap) } var bootstrapInfos []peer.PeerInfo for _, n := range servers { info := n.Peerstore.PeerInfo(n.PeerHost.ID()) bootstrapInfos = append(bootstrapInfos, info) } var clients []*core.IpfsNode for i := 0; i < numClients; i++ { n, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), Routing: corerouting.SupernodeClient(bootstrapInfos...), }) if err != nil { return nil, nil, err } clients = append(clients, n) } mn.LinkAll() bcfg := core.BootstrapConfigWithPeers(bootstrapInfos) for _, n := range clients { if err := n.Bootstrap(bcfg); err != nil { return nil, nil, err } } return servers, clients, nil }
func (i *cmdInvocation) constructNodeFunc(ctx context.Context) func() (*core.IpfsNode, error) { return func() (*core.IpfsNode, error) { if i.req == nil { return nil, errors.New("constructing node without a request") } cmdctx := i.req.InvocContext() if cmdctx == nil { return nil, errors.New("constructing node without a request context") } r, err := fsrepo.Open(i.req.InvocContext().ConfigRoot) if err != nil { // repo is owned by the node return nil, err } // ok everything is good. set it on the invocation (for ownership) // and return it. n, err := core.NewNode(ctx, &core.BuildCfg{ Online: cmdctx.Online, Repo: r, }) if err != nil { return nil, err } i.node = n return i.node, nil } }
func addDefaultAssets(out io.Writer, repoRoot string) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() r, err := fsrepo.Open(repoRoot) if err != nil { // NB: repo is owned by the node return err } nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r}) if err != nil { return err } defer nd.Close() dkey, err := assets.SeedInitDocs(nd) if err != nil { return fmt.Errorf("init: seeding init docs failed: %s", err) } log.Debugf("init: seeded init docs %s", dkey) if _, err = fmt.Fprintf(out, "to get started, enter:\n"); err != nil { return err } _, err = fmt.Fprintf(out, "\n\tipfs cat /ipfs/%s/readme\n\n", dkey) return err }
func MockCmdsCtx() (commands.Context, error) { // Generate Identity ident, err := testutil.RandIdentity() if err != nil { return commands.Context{}, err } p := ident.ID() conf := config.Config{ Identity: config.Identity{ PeerID: p.String(), }, } r := &repo.Mock{ D: ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())), C: conf, } node, err := core.NewNode(context.Background(), &core.BuildCfg{ Repo: r, }) return commands.Context{ Online: true, ConfigRoot: "/tmp/.mockipfsconfig", LoadConfig: func(path string) (*config.Config, error) { return &conf, nil }, ConstructNode: func() (*core.IpfsNode, error) { return node, nil }, }, nil }
// NewMockNode constructs an IpfsNode for use in tests. func NewMockNode() (*core.IpfsNode, error) { ctx := context.Background() // effectively offline, only peer in its network return core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: MockHostOption(mocknet.New(ctx)), }) }
func runFileCattingWorker(ctx context.Context, n *core.IpfsNode) error { conf, err := config.Init(ioutil.Discard, *nBitsForKeypair) if err != nil { return err } r := &repo.Mock{ D: ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())), C: *conf, } dummy, err := core.NewNode(ctx, &core.BuildCfg{ Repo: r, }) if err != nil { return err } go func() { defer dummy.Close() var i int64 = 1 for { buf := new(bytes.Buffer) if err := random.WritePseudoRandomBytes(sizeOfIthFile(i), buf, *seed); err != nil { log.Fatal(err) } // add to a dummy node to discover the key k, err := coreunix.Add(dummy, bytes.NewReader(buf.Bytes())) if err != nil { log.Fatal(err) } e := elog.EventBegin(ctx, "cat", logging.LoggableF(func() map[string]interface{} { return map[string]interface{}{ "key": k, "localPeer": n.Identity, } })) if r, err := coreunix.Cat(n, k); err != nil { e.Done() log.Printf("failed to cat file. seed: %d #%d key: %s err: %s", *seed, i, k, err) } else { log.Println("found file", "seed", *seed, "#", i, "key", k, "size", unit.Information(sizeOfIthFile(i))) io.Copy(ioutil.Discard, r) e.Done() log.Println("catted file", "seed", *seed, "#", i, "key", k, "size", unit.Information(sizeOfIthFile(i))) i++ } time.Sleep(time.Second) } }() return nil }
func newNodeWithMockNamesys(ns mockNamesys) (*core.IpfsNode, error) { c := config.Config{ Identity: config.Identity{ PeerID: "Qmfoo", // required by offline node }, } r := &repo.Mock{ C: c, D: testutil.ThreadSafeCloserMapDatastore(), } n, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r}) if err != nil { return nil, err } n.Namesys = ns return n, nil }
func initializeIpnsKeyspace(repoRoot string) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() r, err := fsrepo.Open(repoRoot) if err != nil { // NB: repo is owned by the node return err } nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r}) if err != nil { return err } defer nd.Close() err = nd.SetupOfflineRouting() if err != nil { return err } return namesys.InitializeKeyspace(ctx, nd.DAG, nd.Namesys, nd.Pinning, nd.PrivateKey) }
func TestAddRecursive(t *testing.T) { here, err := os.Getwd() if err != nil { t.Fatal(err) } r := &repo.Mock{ C: config.Config{ Identity: config.Identity{ PeerID: "Qmfoo", // required by offline node }, }, D: testutil.ThreadSafeCloserMapDatastore(), } node, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r}) if err != nil { t.Fatal(err) } if k, err := AddR(node, path.Join(here, "test_data")); err != nil { t.Fatal(err) } else if k != "QmWCCga8AbTyfAQ7pTnGT6JgmRMAB3Qp8ZmTEFi5q5o8jC" { t.Fatal("keys do not match") } }
func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) { maybeSkipFuseTests(t) var err error if node == nil { node, err = core.NewNode(context.Background(), nil) if err != nil { t.Fatal(err) } err = node.LoadPrivateKey() if err != nil { t.Fatal(err) } node.Routing = offroute.NewOfflineRouter(node.Repo.Datastore(), node.PrivateKey) node.Namesys = namesys.NewNameSystem(node.Routing) ipnsfs, err := nsfs.NewFilesystem(context.Background(), node.DAG, node.Namesys, node.Pinning, node.PrivateKey) if err != nil { t.Fatal(err) } node.IpnsFs = ipnsfs } fs, err := NewFileSystem(node, node.PrivateKey, "", "") if err != nil { t.Fatal(err) } mnt, err := fstest.MountedT(t, fs) if err != nil { t.Fatal(err) } return node, mnt }
func TestBitswapWithoutRouting(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() const numPeers = 4 // create network mn := mocknet.New(ctx) var nodes []*core.IpfsNode for i := 0; i < numPeers; i++ { n, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: coremock.MockHostOption(mn), Routing: core.NilRouterOption, // no routing }) if err != nil { t.Fatal(err) } defer n.Close() nodes = append(nodes, n) } mn.LinkAll() // connect them for _, n1 := range nodes { for _, n2 := range nodes { if n1 == n2 { continue } log.Debug("connecting to other hosts") p2 := n2.PeerHost.Peerstore().PeerInfo(n2.PeerHost.ID()) if err := n1.PeerHost.Connect(ctx, p2); err != nil { t.Fatal(err) } } } // add blocks to each before log.Debug("adding block.") block0 := blocks.NewBlock([]byte("block0")) block1 := blocks.NewBlock([]byte("block1")) // put 1 before if err := nodes[0].Blockstore.Put(block0); err != nil { t.Fatal(err) } // get it out. for i, n := range nodes { // skip first because block not in its exchange. will hang. if i == 0 { continue } log.Debugf("%d %s get block.", i, n.Identity) b, err := n.Blocks.GetBlock(ctx, block0.Key()) if err != nil { t.Error(err) } else if !bytes.Equal(b.Data, block0.Data) { t.Error("byte comparison fail") } else { log.Debug("got block: %s", b.Key()) } } // put 1 after if err := nodes[1].Blockstore.Put(block1); err != nil { t.Fatal(err) } // get it out. for _, n := range nodes { b, err := n.Blocks.GetBlock(ctx, block1.Key()) if err != nil { t.Error(err) } else if !bytes.Equal(b.Data, block1.Data) { t.Error("byte comparison fail") } else { log.Debug("got block: %s", b.Key()) } } }
func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() const numPeers = 3 // create network mn := mocknet.New(ctx) mn.SetLinkDefaults(mocknet.LinkOptions{ Latency: conf.NetworkLatency, // TODO add to conf. This is tricky because we want 0 values to be functional. Bandwidth: math.MaxInt32, }) bootstrap, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer bootstrap.Close() adder, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer adder.Close() catter, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer catter.Close() mn.LinkAll() bis := bootstrap.Peerstore.PeerInfo(bootstrap.PeerHost.ID()) bcfg := core.BootstrapConfigWithPeers([]peer.PeerInfo{bis}) if err := adder.Bootstrap(bcfg); err != nil { return err } if err := catter.Bootstrap(bcfg); err != nil { return err } added, err := coreunix.Add(adder, bytes.NewReader(data)) if err != nil { return err } readerCatted, err := coreunix.Cat(catter, added) if err != nil { return err } // verify bufout := new(bytes.Buffer) io.Copy(bufout, readerCatted) if 0 != bytes.Compare(bufout.Bytes(), data) { return errors.New("catted data does not match added data") } return nil }
func run() error { servers := config.DefaultSNRServers fmt.Println("using gcr remotes:") for _, p := range servers { fmt.Println("\t", p) } ctx, cancel := context.WithCancel(context.Background()) defer cancel() cwd, err := os.Getwd() if err != nil { return err } repoPath := gopath.Join(cwd, config.DefaultPathName) if err := ensureRepoInitialized(repoPath); err != nil { } repo, err := fsrepo.Open(repoPath) if err != nil { // owned by node return err } cfg, err := repo.Config() if err != nil { return err } cfg.Bootstrap = servers if err := repo.SetConfig(cfg); err != nil { return err } var addrs []ipfsaddr.IPFSAddr for _, info := range servers { addr, err := ipfsaddr.ParseString(info) if err != nil { return err } addrs = append(addrs, addr) } var infos []peer.PeerInfo for _, addr := range addrs { infos = append(infos, peer.PeerInfo{ ID: addr.ID(), Addrs: []ma.Multiaddr{addr.Transport()}, }) } node, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Repo: repo, Routing: corerouting.SupernodeClient(infos...), }) if err != nil { return err } defer node.Close() opts := []corehttp.ServeOption{ corehttp.CommandsOption(cmdCtx(node, repoPath)), corehttp.GatewayOption(false), } if *cat { if err := runFileCattingWorker(ctx, node); err != nil { return err } } else { if err := runFileAddingWorker(node); err != nil { return err } } return corehttp.ListenAndServe(node, cfg.Addresses.API, opts...) }
func benchCat(b *testing.B, data []byte, conf testutil.LatencyConfig) error { b.StopTimer() ctx, cancel := context.WithCancel(context.Background()) defer cancel() // create network mn := mocknet.New(ctx) mn.SetLinkDefaults(mocknet.LinkOptions{ Latency: conf.NetworkLatency, // TODO add to conf. This is tricky because we want 0 values to be functional. Bandwidth: math.MaxInt32, }) adder, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer adder.Close() catter, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer catter.Close() err = mn.LinkAll() if err != nil { return err } bs1 := []peer.PeerInfo{adder.Peerstore.PeerInfo(adder.Identity)} bs2 := []peer.PeerInfo{catter.Peerstore.PeerInfo(catter.Identity)} if err := catter.Bootstrap(core.BootstrapConfigWithPeers(bs1)); err != nil { return err } if err := adder.Bootstrap(core.BootstrapConfigWithPeers(bs2)); err != nil { return err } added, err := coreunix.Add(adder, bytes.NewReader(data)) if err != nil { return err } b.StartTimer() readerCatted, err := coreunix.Cat(catter, added) if err != nil { return err } // verify bufout := new(bytes.Buffer) io.Copy(bufout, readerCatted) if 0 != bytes.Compare(bufout.Bytes(), data) { return errors.New("catted data does not match added data") } return nil }
func daemonFunc(req cmds.Request, res cmds.Response) { // let the user know we're going. fmt.Printf("Initializing daemon...\n") ctx := req.InvocContext() go func() { select { case <-req.Context().Done(): fmt.Println("Received interrupt signal, shutting down...") } }() // check transport encryption flag. unencrypted, _, _ := req.Option(unencryptTransportKwd).Bool() if unencrypted { log.Warningf(`Running with --%s: All connections are UNENCRYPTED. You will not be able to connect to regular encrypted networks.`, unencryptTransportKwd) conn.EncryptConnections = false } // first, whether user has provided the initialization flag. we may be // running in an uninitialized state. initialize, _, err := req.Option(initOptionKwd).Bool() if err != nil { res.SetError(err, cmds.ErrNormal) return } if initialize { // now, FileExists is our best method of detecting whether IPFS is // configured. Consider moving this into a config helper method // `IsInitialized` where the quality of the signal can be improved over // time, and many call-sites can benefit. if !util.FileExists(req.InvocContext().ConfigRoot) { err := initWithDefaults(os.Stdout, req.InvocContext().ConfigRoot) if err != nil { res.SetError(err, cmds.ErrNormal) return } } } // acquire the repo lock _before_ constructing a node. we need to make // sure we are permitted to access the resources (datastore, etc.) repo, err := fsrepo.Open(req.InvocContext().ConfigRoot) if err != nil { res.SetError(err, cmds.ErrNormal) return } cfg, err := ctx.GetConfig() if err != nil { res.SetError(err, cmds.ErrNormal) return } // Start assembling node config ncfg := &core.BuildCfg{ Online: true, Repo: repo, } routingOption, _, err := req.Option(routingOptionKwd).String() if err != nil { res.SetError(err, cmds.ErrNormal) return } if routingOption == routingOptionSupernodeKwd { servers, err := cfg.SupernodeRouting.ServerIPFSAddrs() if err != nil { res.SetError(err, cmds.ErrNormal) repo.Close() // because ownership hasn't been transferred to the node return } var infos []peer.PeerInfo for _, addr := range servers { infos = append(infos, peer.PeerInfo{ ID: addr.ID(), Addrs: []ma.Multiaddr{addr.Transport()}, }) } ncfg.Routing = corerouting.SupernodeClient(infos...) } node, err := core.NewNode(req.Context(), ncfg) if err != nil { log.Error("error from node construction: ", err) res.SetError(err, cmds.ErrNormal) return } printSwarmAddrs(node) defer func() { // We wait for the node to close first, as the node has children // that it will wait for before closing, such as the API server. node.Close() select { case <-req.Context().Done(): log.Info("Gracefully shut down daemon") default: } }() req.InvocContext().ConstructNode = func() (*core.IpfsNode, error) { return node, nil } // construct api endpoint - every time err, apiErrc := serveHTTPApi(req) if err != nil { res.SetError(err, cmds.ErrNormal) return } // construct http gateway - if it is set in the config var gwErrc <-chan error if len(cfg.Addresses.Gateway) > 0 { var err error err, gwErrc = serveHTTPGateway(req) if err != nil { res.SetError(err, cmds.ErrNormal) return } } // construct fuse mountpoints - if the user provided the --mount flag mount, _, err := req.Option(mountKwd).Bool() if err != nil { res.SetError(err, cmds.ErrNormal) return } if mount { if err := mountFuse(req); err != nil { res.SetError(err, cmds.ErrNormal) return } } fmt.Printf("Daemon is ready\n") // collect long-running errors and block for shutdown // TODO(cryptix): our fuse currently doesnt follow this pattern for graceful shutdown for err := range merge(apiErrc, gwErrc) { if err != nil { res.SetError(err, cmds.ErrNormal) return } } }
func run(ipfsPath, watchPath string) error { proc := process.WithParent(process.Background()) log.Printf("running IPFSWatch on '%s' using repo at '%s'...", watchPath, ipfsPath) ipfsPath, err := homedir.Expand(ipfsPath) if err != nil { return err } watcher, err := fsnotify.NewWatcher() if err != nil { return err } defer watcher.Close() if err := addTree(watcher, watchPath); err != nil { return err } r, err := fsrepo.Open(ipfsPath) if err != nil { // TODO handle case: daemon running // TODO handle case: repo doesn't exist or isn't initialized return err } node, err := core.NewNode(context.Background(), &core.BuildCfg{ Online: true, Repo: r, }) if err != nil { return err } defer node.Close() if *http { addr := "/ip4/127.0.0.1/tcp/5001" var opts = []corehttp.ServeOption{ corehttp.GatewayOption(true), corehttp.WebUIOption, corehttp.CommandsOption(cmdCtx(node, ipfsPath)), } proc.Go(func(p process.Process) { if err := corehttp.ListenAndServe(node, addr, opts...); err != nil { return } }) } interrupts := make(chan os.Signal) signal.Notify(interrupts, os.Interrupt, os.Kill) for { select { case <-interrupts: return nil case e := <-watcher.Events: log.Printf("received event: %s", e) isDir, err := IsDirectory(e.Name) if err != nil { continue } switch e.Op { case fsnotify.Remove: if isDir { if err := watcher.Remove(e.Name); err != nil { return err } } default: // all events except for Remove result in an IPFS.Add, but only // directory creation triggers a new watch switch e.Op { case fsnotify.Create: if isDir { addTree(watcher, e.Name) } } proc.Go(func(p process.Process) { file, err := os.Open(e.Name) if err != nil { log.Println(err) } defer file.Close() k, err := coreunix.Add(node, file) if err != nil { log.Println(err) } log.Printf("added %s... key: %s", e.Name, k) }) } case err := <-watcher.Errors: log.Println(err) } } return nil }
res.SetError(err, cmds.ErrNormal) return } progress, _, _ := req.Option(progressOptionName).Bool() trickle, _, _ := req.Option(trickleOptionName).Bool() wrap, _, _ := req.Option(wrapOptionName).Bool() hash, _, _ := req.Option(onlyHashOptionName).Bool() hidden, _, _ := req.Option(hiddenOptionName).Bool() chunker, _, _ := req.Option(chunkerOptionName).String() e := dagutils.NewDagEditor(NewMemoryDagService(), newDirNode()) if hash { nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{ //TODO: need this to be true or all files // hashed will be stored in memory! NilRepo: true, }) if err != nil { res.SetError(err, cmds.ErrNormal) return } n = nilnode } outChan := make(chan interface{}, 8) res.SetOutput((<-chan interface{})(outChan)) fileAdder := adder{ ctx: req.Context(), node: n,