func InitializeSupernodeNetwork(
ctx context.Context,
numServers, numClients int,
conf testutil.LatencyConfig) ([]*core.IpfsNode, []*core.IpfsNode, error) {
// create network
mn := mocknet.New(ctx)
mn.SetLinkDefaults(mocknet.LinkOptions{
Latency: conf.NetworkLatency,
Bandwidth: math.MaxInt32,
})
routingDatastore := ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore()))
var servers []*core.IpfsNode
for i := 0; i < numServers; i++ {
bootstrap, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
Routing: corerouting.SupernodeServer(routingDatastore),
})
if err != nil {
return nil, nil, err
}
servers = append(servers, bootstrap)
}
var bootstrapInfos []pstore.PeerInfo
for _, n := range servers {
info := n.Peerstore.PeerInfo(n.PeerHost.ID())
bootstrapInfos = append(bootstrapInfos, info)
}
var clients []*core.IpfsNode
for i := 0; i < numClients; i++ {
n, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
Routing: corerouting.SupernodeClient(bootstrapInfos...),
})
if err != nil {
return nil, nil, err
}
clients = append(clients, n)
}
mn.LinkAll()
bcfg := core.BootstrapConfigWithPeers(bootstrapInfos)
for _, n := range clients {
if err := n.Bootstrap(bcfg); err != nil {
return nil, nil, err
}
}
return servers, clients, nil
}
func initializeIpnsKeyspace(repoRoot string, privKeyBytes []byte) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
r, err := fsrepo.Open(repoRoot)
if err != nil { // NB: repo is owned by the node
return err
}
cfg, err := r.Config()
if err != nil {
log.Error(err)
return err
}
identity, err := ipfs.IdentityFromKey(privKeyBytes)
if err != nil {
return err
}
cfg.Identity = identity
nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r})
if err != nil {
return err
}
defer nd.Close()
err = nd.SetupOfflineRouting()
if err != nil {
return err
}
return namesys.InitializeKeyspace(ctx, nd.DAG, nd.Namesys, nd.Pinning, nd.PrivateKey)
}
func startupIPFS(dspath string, ctx *context.Context) (*core.IpfsNode, error) {
r, err := fsrepo.Open(dspath)
if err != nil {
config, err := config.Init(os.Stdout, 2048)
if err != nil {
return nil, err
}
if err := fsrepo.Init(dspath, config); err != nil {
return nil, err
}
r, err = fsrepo.Open(dspath)
if err != nil {
return nil, err
}
}
//fmt.Println(r)
node, err := core.NewNode(*ctx, &core.BuildCfg{Online: true, Repo: r})
if err != nil {
return nil, err
}
return node, nil
}
func NewTmpDirNode(ctx context.Context) (*core.IpfsNode, error) {
dir, err := ioutil.TempDir("", "ipfs-shell")
if err != nil {
return nil, fmt.Errorf("failed to get temp dir: %s", err)
}
cfg, err := config.Init(ioutil.Discard, 1024)
if err != nil {
return nil, err
}
err = fsrepo.Init(dir, cfg)
if err != nil {
return nil, fmt.Errorf("failed to init ephemeral node: %s", err)
}
repo, err := fsrepo.Open(dir)
if err != nil {
return nil, err
}
return core.NewNode(ctx, &core.BuildCfg{
Online: true,
Repo: repo,
})
}
func BuildNode(ctx context.Context, dcConfig *DCConfig) (*core.IpfsNode, error) {
// Get a IPFS repo/config directory, and open it
ipfsPath, err := getIpfsPath(dcConfig.PoolIndex)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
repo, err := fsrepo.Open(ipfsPath)
if err != nil {
return nil, err
}
// Create the node with a config
node, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Repo: repo,
})
if err != nil {
return nil, err
}
return node, nil
}
func MockCmdsCtx() (commands.Context, error) {
// Generate Identity
ident, err := testutil.RandIdentity()
if err != nil {
return commands.Context{}, err
}
p := ident.ID()
conf := config.Config{
Identity: config.Identity{
PeerID: p.String(),
},
}
r := &repo.Mock{
D: ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())),
C: conf,
}
node, err := core.NewNode(context.Background(), &core.BuildCfg{
Repo: r,
})
return commands.Context{
Online: true,
ConfigRoot: "/tmp/.mockipfsconfig",
LoadConfig: func(path string) (*config.Config, error) {
return &conf, nil
},
ConstructNode: func() (*core.IpfsNode, error) {
return node, nil
},
}, nil
}
func (i *cmdInvocation) constructNodeFunc(ctx context.Context) func() (*core.IpfsNode, error) {
return func() (*core.IpfsNode, error) {
if i.req == nil {
return nil, errors.New("constructing node without a request")
}
cmdctx := i.req.InvocContext()
if cmdctx == nil {
return nil, errors.New("constructing node without a request context")
}
r, err := fsrepo.Open(i.req.InvocContext().ConfigRoot)
if err != nil { // repo is owned by the node
return nil, err
}
// ok everything is good. set it on the invocation (for ownership)
// and return it.
n, err := core.NewNode(ctx, &core.BuildCfg{
Online: cmdctx.Online,
Repo: r,
})
if err != nil {
return nil, err
}
i.node = n
return i.node, nil
}
}
func addDefaultAssets(out io.Writer, repoRoot string) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
r, err := fsrepo.Open(repoRoot)
if err != nil { // NB: repo is owned by the node
return err
}
nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r})
if err != nil {
return err
}
defer nd.Close()
dkey, err := assets.SeedInitDocs(nd)
if err != nil {
return fmt.Errorf("init: seeding init docs failed: %s", err)
}
log.Debugf("init: seeded init docs %s", dkey)
if _, err = fmt.Fprintf(out, "to get started, enter:\n"); err != nil {
return err
}
_, err = fmt.Fprintf(out, "\n\tipfs cat /ipfs/%s/readme\n\n", dkey)
return err
}
func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) {
maybeSkipFuseTests(t)
var err error
if node == nil {
node, err = core.NewNode(context.Background(), nil)
if err != nil {
t.Fatal(err)
}
err = node.LoadPrivateKey()
if err != nil {
t.Fatal(err)
}
node.Routing = offroute.NewOfflineRouter(node.Repo.Datastore(), node.PrivateKey)
node.Namesys = namesys.NewNameSystem(node.Routing, node.Repo.Datastore(), 0)
err = InitializeKeyspace(node, node.PrivateKey)
if err != nil {
t.Fatal(err)
}
}
fs, err := NewFileSystem(node, node.PrivateKey, "", "")
if err != nil {
t.Fatal(err)
}
mnt, err := fstest.MountedT(t, fs)
if err != nil {
t.Fatal(err)
}
return node, mnt
}
func createNode(nd *Node, online bool, ctx context.Context) (*core.IpfsNode, error) {
// `nd` only contains the prepopulated fields as in New().
rp, err := fsrepo.Open(nd.Path)
if err != nil {
log.Errorf("Unable to open repo `%s`: %v", nd.Path, err)
return nil, err
}
swarmAddrs := []string{
fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", nd.SwarmPort),
fmt.Sprintf("/ip6/::/tcp/%d", nd.SwarmPort),
}
if err := rp.SetConfigKey("Addresses.Swarm", swarmAddrs); err != nil {
return nil, err
}
cfg := &core.BuildCfg{
Repo: rp,
Online: online,
}
ipfsNode, err := core.NewNode(ctx, cfg)
if err != nil {
return nil, err
}
return ipfsNode, nil
}
func runFileCattingWorker(ctx context.Context, n *core.IpfsNode) error {
conf, err := config.Init(ioutil.Discard, *nBitsForKeypair)
if err != nil {
return err
}
r := &repo.Mock{
D: ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())),
C: *conf,
}
dummy, err := core.NewNode(ctx, &core.BuildCfg{
Repo: r,
})
if err != nil {
return err
}
errs := make(chan error)
go func() {
defer dummy.Close()
var i int64 = 1
for {
buf := new(bytes.Buffer)
if err := random.WritePseudoRandomBytes(sizeOfIthFile(i), buf, *seed); err != nil {
errs <- err
}
// add to a dummy node to discover the key
k, err := coreunix.Add(dummy, bytes.NewReader(buf.Bytes()))
if err != nil {
errs <- err
}
e := elog.EventBegin(ctx, "cat", logging.LoggableF(func() map[string]interface{} {
return map[string]interface{}{
"key": k,
"localPeer": n.Identity,
}
}))
if r, err := coreunix.Cat(ctx, n, k); err != nil {
e.Done()
log.Printf("failed to cat file. seed: %d #%d key: %s err: %s", *seed, i, k, err)
} else {
log.Println("found file", "seed", *seed, "#", i, "key", k, "size", unit.Information(sizeOfIthFile(i)))
io.Copy(ioutil.Discard, r)
e.Done()
log.Println("catted file", "seed", *seed, "#", i, "key", k, "size", unit.Information(sizeOfIthFile(i)))
i++
}
time.Sleep(time.Second)
}
}()
err = <-errs
if err != nil {
log.Fatal(err)
}
return nil
}
// NewMockNode constructs an IpfsNode for use in tests.
func NewMockNode() (*core.IpfsNode, error) {
ctx := context.Background()
// effectively offline, only peer in its network
return core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: MockHostOption(mocknet.New(ctx)),
})
}
//NewSelf make repo if needed ,starts daemon and returns Self obj.
func NewSelf(cfg *config.Config, rootPath string) *Self {
InitRepo(cfg.IpfsRepo)
n := &merkledag.Node{}
k, err := n.Key()
log.IfFatal(err)
//workaround "ERROR bitswap: failed to find any peer in table"
i := 0
var node *core.IpfsNode
var ctx context.Context
var cancel context.CancelFunc
for i = 0; i < 10; i++ {
log.Println("setting up node...")
r, err := fsrepo.Open(cfg.IpfsRepo)
log.IfFatal(err)
ctx, cancel = context.WithCancel(context.Background())
node, err = core.NewNode(ctx, &core.BuildCfg{
Repo: r,
Online: true,
})
log.IfFatal(err)
if err := node.Routing.Provide(ctx, k); log.If(err) {
cancel()
log.If(node.Close())
log.Println("retrying...")
continue
}
break
}
if i == 10 {
log.Fatal("cannot provide a key to network")
}
self := &Self{
RootPath: rootPath,
ipfsNode: node,
ctx: ctx,
cancel: cancel,
cfg: cfg,
}
self.follow = FromStringSlice(cfg.FollowPeers, self)
parent, err := self.ToPeer().GetDAGNode("")
if log.If(err) {
parent = &merkledag.Node{}
}
self.myIpns, err = parent.Key()
log.IfFatal(err)
if _, err = parent.GetNodeLink(rootPath); err != nil {
log.Println("initializing DAGs for saving status")
self.makeInitNodes(parent)
}
return self
}
func SetupIpfs() (*core.IpfsNode, error) {
// Assume the user has run 'ipfs init'
r, err := fsrepo.Open("~/.ipfs")
if err != nil {
return nil, err
}
cfg := &core.BuildCfg{
Repo: r,
Online: true,
}
return core.NewNode(context.Background(), cfg)
}
func testAddWPosInfo(t *testing.T, rawLeaves bool) {
r := &repo.Mock{
C: config.Config{
Identity: config.Identity{
PeerID: "Qmfoo", // required by offline node
},
},
D: testutil.ThreadSafeCloserMapDatastore(),
}
node, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
if err != nil {
t.Fatal(err)
}
bs := &testBlockstore{GCBlockstore: node.Blockstore, expectedPath: "/tmp/foo.txt", t: t}
bserv := blockservice.New(bs, node.Exchange)
dserv := dag.NewDAGService(bserv)
adder, err := NewAdder(context.Background(), node.Pinning, bs, dserv)
if err != nil {
t.Fatal(err)
}
adder.Out = make(chan interface{})
adder.Progress = true
adder.RawLeaves = rawLeaves
data := make([]byte, 5*1024*1024)
rand.New(rand.NewSource(2)).Read(data) // Rand.Read never returns an error
fileData := ioutil.NopCloser(bytes.NewBuffer(data))
fileInfo := dummyFileInfo{"foo.txt", int64(len(data)), time.Now()}
file := files.NewReaderFile("foo.txt", "/tmp/foo.txt", fileData, &fileInfo)
go func() {
defer close(adder.Out)
err = adder.AddFile(file)
if err != nil {
t.Fatal(err)
}
}()
for _ = range adder.Out {
}
if bs.countAtOffsetZero != 2 {
t.Fatal("expected 2 blocks with an offset at zero (one root and one leafh), got", bs.countAtOffsetZero)
}
if bs.countAtOffsetNonZero != 19 {
// note: the exact number will depend on the size and the sharding algo. used
t.Fatal("expected 19 blocks with an offset > 0, got", bs.countAtOffsetNonZero)
}
}
func main() {
bsize := kingpin.Flag("blocksize", "blocksize to test with").Default("262144").Int64()
kingpin.Parse()
ipfsdir := getIpfsDir()
r, err := fsrepo.Open(ipfsdir)
if err != nil {
fmt.Printf("Failed to open ipfs repo at: %s: %s\n", ipfsdir, err)
fmt.Println("Please ensure ipfs has been initialized and that no daemon is running")
return
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
nd, err := core.NewNode(ctx, &core.BuildCfg{
Repo: r,
})
if err != nil {
fmt.Printf("failed to create node: %s\n", err)
return
}
cfg := &BenchCfg{
Blocksize: *bsize,
}
fmt.Println(cfg)
err = BenchmarkBlockRewrites(nd, cfg)
if err != nil {
panic(err)
}
err = BenchmarkRandomBlockWrites(nd, cfg)
if err != nil {
panic(err)
}
err = BenchmarkAdd(nd, cfg)
if err != nil {
panic(err)
}
err = BenchmarkDiskWrites(ipfsdir)
if err != nil {
panic(err)
}
}
func makeAPI(ctx context.Context) (*core.IpfsNode, coreiface.UnixfsAPI, error) {
r := &repo.Mock{
C: config.Config{
Identity: config.Identity{
PeerID: "Qmfoo", // required by offline node
},
},
D: testutil.ThreadSafeCloserMapDatastore(),
}
node, err := core.NewNode(ctx, &core.BuildCfg{Repo: r})
if err != nil {
return nil, nil, err
}
api := coreapi.NewUnixfsAPI(node)
return node, api, nil
}
func newNodeWithMockNamesys(ns mockNamesys) (*core.IpfsNode, error) {
c := config.Config{
Identity: config.Identity{
PeerID: "Qmfoo", // required by offline node
},
}
r := &repo.Mock{
C: c,
D: testutil.ThreadSafeCloserMapDatastore(),
}
n, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
if err != nil {
return nil, err
}
n.Namesys = ns
return n, nil
}
func TestAddRecursive(t *testing.T) {
r := &repo.Mock{
C: config.Config{
Identity: config.Identity{
PeerID: "Qmfoo", // required by offline node
},
},
D: testutil.ThreadSafeCloserMapDatastore(),
}
node, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
if err != nil {
t.Fatal(err)
}
if k, err := AddR(node, "test_data"); err != nil {
t.Fatal(err)
} else if k != "QmWCCga8AbTyfAQ7pTnGT6JgmRMAB3Qp8ZmTEFi5q5o8jC" {
t.Fatal("keys do not match: ", k)
}
}
func NewDefaultNodeWithFSRepo(ctx context.Context, repoPath string) (*core.IpfsNode, error) {
r, err := fsrepo.Open(repoPath)
if err != nil {
return nil, errgo.Notef(err, "opening fsrepo failed")
}
node, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Repo: r,
})
if err != nil {
return nil, errgo.Notef(err, "ipfs NewNode() failed.")
}
// TODO: can we bootsrap localy/mdns first and fall back to default?
err = node.Bootstrap(core.DefaultBootstrapConfig)
if err != nil {
return nil, errgo.Notef(err, "ipfs Bootstrap() failed.")
}
return node, nil
}
func main() {
if len(os.Args) < 2 {
fmt.Println("Please give a peer ID as an argument")
return
}
target, err := peer.IDB58Decode(os.Args[1])
if err != nil {
panic(err)
}
// Basic ipfsnode setup
r, err := fsrepo.Open("~/.ipfs")
if err != nil {
panic(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cfg := &core.BuildCfg{
Repo: r,
Online: true,
}
nd, err := core.NewNode(ctx, cfg)
if err != nil {
panic(err)
}
fmt.Printf("I am peer %s dialing %s\n", nd.Identity, target)
con, err := corenet.Dial(nd, target, "/app/whyrusleeping")
if err != nil {
fmt.Println(err)
return
}
io.Copy(os.Stdout, con)
}
func initializeIpnsKeyspace(repoRoot string) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
r, err := fsrepo.Open(repoRoot)
if err != nil { // NB: repo is owned by the node
return err
}
nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r})
if err != nil {
return err
}
defer nd.Close()
err = nd.SetupOfflineRouting()
if err != nil {
return err
}
return namesys.InitializeKeyspace(ctx, nd.DAG, nd.Namesys, nd.Pinning, nd.PrivateKey)
}
func main() {
// Basic ipfsnode setup
r, err := fsrepo.Open("~/.ipfs")
if err != nil {
panic(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cfg := &core.BuildCfg{
Repo: r,
Online: true,
}
nd, err := core.NewNode(ctx, cfg)
if err != nil {
panic(err)
}
list, err := corenet.Listen(nd, "/app/whyrusleeping")
if err != nil {
panic(err)
}
fmt.Printf("I am peer: %s\n", nd.Identity.Pretty())
for {
con, err := list.Accept()
if err != nil {
fmt.Println(err)
return
}
defer con.Close()
fmt.Fprintln(con, "Hello! This is whyrusleepings awesome ipfs service")
fmt.Printf("Connection from: %s\n", con.Conn().RemotePeer())
}
}
func get(path, outFile string) {
node, err := core.NewNode(context.Background(), &core.BuildCfg{
Online: true,
})
if err != nil {
panic(err)
}
err = node.Bootstrap(core.DefaultBootstrapConfig)
if err != nil {
fmt.Printf("%v\n", err)
return
}
reader, length, err := cat(node.Context(), node, path)
if err != nil {
fmt.Printf("%v\n", err)
return
}
file, err := os.Create(outFile)
if err != nil {
fmt.Printf("%v", err)
return
}
bar := pb.New(int(length)).SetUnits(pb.U_BYTES)
bar.ShowSpeed = false
bar.Start()
writer := io.MultiWriter(file, bar)
io.Copy(writer, reader)
bar.Finish()
fmt.Printf("wrote %v to %v (%v bytes)\n", path, outFile, length)
}
func addDefaultAssets(out io.Writer, repoRoot string) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
r, err := fsrepo.Open(repoRoot)
if err != nil { // NB: repo is owned by the node
return err
}
nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r})
if err != nil {
return err
}
defer func() {
log.If(nd.Close())
}()
dkey, err := assets.SeedInitDocs(nd)
if err != nil {
return fmt.Errorf("init: seeding init docs failed: %s", err)
}
log.Println("init: seeded init docs", dkey)
return nil
}
// res.SetError(err, cmds.ErrNormal)
// return
//}
progress, _, _ := req.Option(progressOptionName).Bool()
trickle, _, _ := req.Option(trickleOptionName).Bool()
wrap, _, _ := req.Option(wrapOptionName).Bool()
hash, _, _ := req.Option(onlyHashOptionName).Bool()
hidden, _, _ := req.Option(hiddenOptionName).Bool()
chunker, _, _ := req.Option(chunkerOptionName).String()
e := dagutils.NewDagEditor(NewMemoryDagService(), newDirNode())
if hash {
nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
//TODO: need this to be true or all files
// hashed will be stored in memory!
NilRepo: true,
})
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
n = nilnode
}
outChan := make(chan interface{}, 8)
res.SetOutput((<-chan interface{})(outChan))
fileAdder := adder{
ctx: req.Context(),
node: n,
func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
const numPeers = 3
// create network
mn := mocknet.New(ctx)
mn.SetLinkDefaults(mocknet.LinkOptions{
Latency: conf.NetworkLatency,
// TODO add to conf. This is tricky because we want 0 values to be functional.
Bandwidth: math.MaxInt32,
})
bootstrap, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer bootstrap.Close()
adder, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer adder.Close()
catter, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer catter.Close()
mn.LinkAll()
bis := bootstrap.Peerstore.PeerInfo(bootstrap.PeerHost.ID())
bcfg := core.BootstrapConfigWithPeers([]peer.PeerInfo{bis})
if err := adder.Bootstrap(bcfg); err != nil {
return err
}
if err := catter.Bootstrap(bcfg); err != nil {
return err
}
added, err := coreunix.Add(adder, bytes.NewReader(data))
if err != nil {
return err
}
readerCatted, err := coreunix.Cat(ctx, catter, added)
if err != nil {
return err
}
// verify
bufout := new(bytes.Buffer)
io.Copy(bufout, readerCatted)
if 0 != bytes.Compare(bufout.Bytes(), data) {
return errors.New("catted data does not match added data")
}
return nil
}
func run() error {
servers := config.DefaultSNRServers
fmt.Println("using gcr remotes:")
for _, p := range servers {
fmt.Println("\t", p)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cwd, err := os.Getwd()
if err != nil {
return err
}
repoPath := gopath.Join(cwd, config.DefaultPathName)
if err := ensureRepoInitialized(repoPath); err != nil {
}
repo, err := fsrepo.Open(repoPath)
if err != nil { // owned by node
return err
}
cfg, err := repo.Config()
if err != nil {
return err
}
cfg.Bootstrap = servers
if err := repo.SetConfig(cfg); err != nil {
return err
}
var addrs []ipfsaddr.IPFSAddr
for _, info := range servers {
addr, err := ipfsaddr.ParseString(info)
if err != nil {
return err
}
addrs = append(addrs, addr)
}
var infos []peer.PeerInfo
for _, addr := range addrs {
infos = append(infos, peer.PeerInfo{
ID: addr.ID(),
Addrs: []ma.Multiaddr{addr.Transport()},
})
}
node, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Repo: repo,
Routing: corerouting.SupernodeClient(infos...),
})
if err != nil {
return err
}
defer node.Close()
opts := []corehttp.ServeOption{
corehttp.CommandsOption(cmdCtx(node, repoPath)),
corehttp.GatewayOption(false),
}
if *cat {
if err := runFileCattingWorker(ctx, node); err != nil {
return err
}
} else {
if err := runFileAddingWorker(node); err != nil {
return err
}
}
return corehttp.ListenAndServe(node, cfg.Addresses.API, opts...)
}
func TestAddGCLive(t *testing.T) {
r := &repo.Mock{
C: config.Config{
Identity: config.Identity{
PeerID: "Qmfoo", // required by offline node
},
},
D: testutil.ThreadSafeCloserMapDatastore(),
}
node, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
if err != nil {
t.Fatal(err)
}
errs := make(chan error)
out := make(chan interface{})
adder, err := NewAdder(context.Background(), node, out)
if err != nil {
t.Fatal(err)
}
dataa := ioutil.NopCloser(bytes.NewBufferString("testfileA"))
rfa := files.NewReaderFile("a", "a", dataa, nil)
// make two files with pipes so we can 'pause' the add for timing of the test
piper, pipew := io.Pipe()
hangfile := files.NewReaderFile("b", "b", piper, nil)
datad := ioutil.NopCloser(bytes.NewBufferString("testfileD"))
rfd := files.NewReaderFile("d", "d", datad, nil)
slf := files.NewSliceFile("files", "files", []files.File{rfa, hangfile, rfd})
addDone := make(chan struct{})
go func() {
defer close(addDone)
defer close(out)
err := adder.AddFile(slf)
if err != nil {
t.Fatal(err)
}
}()
addedHashes := make(map[string]struct{})
select {
case o := <-out:
addedHashes[o.(*AddedObject).Hash] = struct{}{}
case <-addDone:
t.Fatal("add shouldnt complete yet")
}
var gcout <-chan key.Key
gcstarted := make(chan struct{})
go func() {
defer close(gcstarted)
gcchan, err := gc.GC(context.Background(), node.Blockstore, node.Pinning)
if err != nil {
log.Error("GC ERROR:", err)
errs <- err
return
}
gcout = gcchan
}()
// gc shouldnt start until we let the add finish its current file.
pipew.Write([]byte("some data for file b"))
select {
case <-gcstarted:
t.Fatal("gc shouldnt have started yet")
case err := <-errs:
t.Fatal(err)
default:
}
time.Sleep(time.Millisecond * 100) // make sure gc gets to requesting lock
// finish write and unblock gc
pipew.Close()
// receive next object from adder
select {
case o := <-out:
addedHashes[o.(*AddedObject).Hash] = struct{}{}
case err := <-errs:
t.Fatal(err)
}
select {
case <-gcstarted:
case err := <-errs:
t.Fatal(err)
}
for k := range gcout {
if _, ok := addedHashes[k.B58String()]; ok {
t.Fatal("gc'ed a hash we just added")
}
}
var last key.Key
for a := range out {
// wait for it to finish
last = key.B58KeyDecode(a.(*AddedObject).Hash)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
root, err := node.DAG.Get(ctx, last)
if err != nil {
t.Fatal(err)
}
err = dag.EnumerateChildren(ctx, node.DAG, root, key.NewKeySet())
if err != nil {
t.Fatal(err)
}
}
func benchCat(b *testing.B, data []byte, conf testutil.LatencyConfig) error {
b.StopTimer()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create network
mn := mocknet.New(ctx)
mn.SetLinkDefaults(mocknet.LinkOptions{
Latency: conf.NetworkLatency,
// TODO add to conf. This is tricky because we want 0 values to be functional.
Bandwidth: math.MaxInt32,
})
adder, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer adder.Close()
catter, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer catter.Close()
err = mn.LinkAll()
if err != nil {
return err
}
bs1 := []peer.PeerInfo{adder.Peerstore.PeerInfo(adder.Identity)}
bs2 := []peer.PeerInfo{catter.Peerstore.PeerInfo(catter.Identity)}
if err := catter.Bootstrap(core.BootstrapConfigWithPeers(bs1)); err != nil {
return err
}
if err := adder.Bootstrap(core.BootstrapConfigWithPeers(bs2)); err != nil {
return err
}
added, err := coreunix.Add(adder, bytes.NewReader(data))
if err != nil {
return err
}
b.StartTimer()
readerCatted, err := coreunix.Cat(ctx, catter, added)
if err != nil {
return err
}
// verify
bufout := new(bytes.Buffer)
io.Copy(bufout, readerCatted)
if 0 != bytes.Compare(bufout.Bytes(), data) {
return errors.New("catted data does not match added data")
}
return nil
}