func cat(ctx context.Context, node *core.IpfsNode, paths []string) ([]io.Reader, uint64, error) { readers := make([]io.Reader, 0, len(paths)) length := uint64(0) for _, fpath := range paths { read, err := coreunix.Cat(node, fpath) if err != nil { return nil, 0, err } readers = append(readers, read) length += uint64(read.Size()) } return readers, length, nil }
func runFileCattingWorker(ctx context.Context, n *core.IpfsNode) error { conf, err := config.Init(ioutil.Discard, *nBitsForKeypair) if err != nil { return err } r := &repo.Mock{ D: ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())), C: *conf, } dummy, err := core.NewNode(ctx, &core.BuildCfg{ Repo: r, }) if err != nil { return err } go func() { defer dummy.Close() var i int64 = 1 for { buf := new(bytes.Buffer) if err := random.WritePseudoRandomBytes(sizeOfIthFile(i), buf, *seed); err != nil { log.Fatal(err) } // add to a dummy node to discover the key k, err := coreunix.Add(dummy, bytes.NewReader(buf.Bytes())) if err != nil { log.Fatal(err) } e := elog.EventBegin(ctx, "cat", logging.LoggableF(func() map[string]interface{} { return map[string]interface{}{ "key": k, "localPeer": n.Identity, } })) if r, err := coreunix.Cat(n, k); err != nil { e.Done() log.Printf("failed to cat file. seed: %d #%d key: %s err: %s", *seed, i, k, err) } else { log.Println("found file", "seed", *seed, "#", i, "key", k, "size", unit.Information(sizeOfIthFile(i))) io.Copy(ioutil.Discard, r) e.Done() log.Println("catted file", "seed", *seed, "#", i, "key", k, "size", unit.Information(sizeOfIthFile(i))) i++ } time.Sleep(time.Second) } }() return nil }
func RunSupernodeBootstrappedAddCat(data []byte, conf testutil.LatencyConfig) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() servers, clients, err := InitializeSupernodeNetwork(ctx, 8, 2, conf) if err != nil { return err } for _, n := range append(servers, clients...) { defer n.Close() } adder := clients[0] catter := clients[1] log.Info("adder is", adder.Identity) log.Info("catter is", catter.Identity) keyAdded, err := coreunix.Add(adder, bytes.NewReader(data)) if err != nil { return err } readerCatted, err := coreunix.Cat(catter, keyAdded) if err != nil { return err } // verify bufout := new(bytes.Buffer) io.Copy(bufout, readerCatted) if 0 != bytes.Compare(bufout.Bytes(), data) { return errors.New("catted data does not match added data") } return nil }
func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() const numPeers = 3 // create network mn := mocknet.New(ctx) mn.SetLinkDefaults(mocknet.LinkOptions{ Latency: conf.NetworkLatency, // TODO add to conf. This is tricky because we want 0 values to be functional. Bandwidth: math.MaxInt32, }) bootstrap, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer bootstrap.Close() adder, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer adder.Close() catter, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer catter.Close() mn.LinkAll() bis := bootstrap.Peerstore.PeerInfo(bootstrap.PeerHost.ID()) bcfg := core.BootstrapConfigWithPeers([]peer.PeerInfo{bis}) if err := adder.Bootstrap(bcfg); err != nil { return err } if err := catter.Bootstrap(bcfg); err != nil { return err } added, err := coreunix.Add(adder, bytes.NewReader(data)) if err != nil { return err } readerCatted, err := coreunix.Cat(catter, added) if err != nil { return err } // verify bufout := new(bytes.Buffer) io.Copy(bufout, readerCatted) if 0 != bytes.Compare(bufout.Bytes(), data) { return errors.New("catted data does not match added data") } return nil }
func benchCat(b *testing.B, data []byte, conf testutil.LatencyConfig) error { b.StopTimer() ctx, cancel := context.WithCancel(context.Background()) defer cancel() // create network mn := mocknet.New(ctx) mn.SetLinkDefaults(mocknet.LinkOptions{ Latency: conf.NetworkLatency, // TODO add to conf. This is tricky because we want 0 values to be functional. Bandwidth: math.MaxInt32, }) adder, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer adder.Close() catter, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer catter.Close() err = mn.LinkAll() if err != nil { return err } bs1 := []peer.PeerInfo{adder.Peerstore.PeerInfo(adder.Identity)} bs2 := []peer.PeerInfo{catter.Peerstore.PeerInfo(catter.Identity)} if err := catter.Bootstrap(core.BootstrapConfigWithPeers(bs1)); err != nil { return err } if err := adder.Bootstrap(core.BootstrapConfigWithPeers(bs2)); err != nil { return err } added, err := coreunix.Add(adder, bytes.NewReader(data)) if err != nil { return err } b.StartTimer() readerCatted, err := coreunix.Cat(catter, added) if err != nil { return err } // verify bufout := new(bytes.Buffer) io.Copy(bufout, readerCatted) if 0 != bytes.Compare(bufout.Bytes(), data) { return errors.New("catted data does not match added data") } return nil }
// Perform a large number of concurrent reads to stress the system func TestIpfsStressRead(t *testing.T) { if testing.Short() { t.SkipNow() } nd, mnt := setupIpfsTest(t, nil) defer mnt.Close() var ks []key.Key var paths []string nobj := 50 ndiriter := 50 // Make a bunch of objects for i := 0; i < nobj; i++ { fi, _ := randObj(t, nd, rand.Int63n(50000)) k, err := fi.Key() if err != nil { t.Fatal(err) } ks = append(ks, k) paths = append(paths, k.String()) } // Now make a bunch of dirs for i := 0; i < ndiriter; i++ { db := uio.NewDirectory(nd.DAG) for j := 0; j < 1+rand.Intn(10); j++ { name := fmt.Sprintf("child%d", j) err := db.AddChild(nd.Context(), name, ks[rand.Intn(len(ks))]) if err != nil { t.Fatal(err) } } newdir := db.GetNode() k, err := nd.DAG.Add(newdir) if err != nil { t.Fatal(err) } ks = append(ks, k) npaths := getPaths(t, nd, k.String(), newdir) paths = append(paths, npaths...) } // Now read a bunch, concurrently wg := sync.WaitGroup{} for s := 0; s < 4; s++ { wg.Add(1) go func() { defer wg.Done() for i := 0; i < 2000; i++ { item := paths[rand.Intn(len(paths))] fname := path.Join(mnt.Dir, item) rbuf, err := ioutil.ReadFile(fname) if err != nil { t.Fatal(err) } read, err := coreunix.Cat(nd, item) if err != nil { t.Fatal(err) } data, err := ioutil.ReadAll(read) if err != nil { t.Fatal(err) } if !bytes.Equal(rbuf, data) { t.Fatal("Incorrect Read!") } } }() } wg.Wait() }