func BenchmarkHandle10KBlocks(b *testing.B) { bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) var testdata []*blocks.Block for i := 0; i < 10000; i++ { testdata = append(testdata, blocks.NewBlock([]byte(string(i)))) } b.ResetTimer() b.SetBytes(10000) for i := 0; i < b.N; i++ { b.StopTimer() w := NewWorker(offline.Exchange(bstore), Config{ NumWorkers: 1, ClientBufferSize: 0, WorkerBufferSize: 0, }) b.StartTimer() for _, block := range testdata { if err := w.HasBlock(block); err != nil { b.Fatal(err) } } b.StopTimer() w.Close() b.StartTimer() } }
func (cfg *BuildCfg) fillDefaults() error { if cfg.Repo != nil && cfg.NilRepo { return errors.New("cannot set a repo and specify nilrepo at the same time") } if cfg.Repo == nil { var d ds.Datastore d = ds.NewMapDatastore() if cfg.NilRepo { d = ds.NewNullDatastore() } r, err := defaultRepo(dsync.MutexWrap(d)) if err != nil { return err } cfg.Repo = r } if cfg.Routing == nil { cfg.Routing = DHTOption } if cfg.Host == nil { cfg.Host = DefaultHostOption } return nil }
func MockCmdsCtx() (commands.Context, error) { // Generate Identity ident, err := testutil.RandIdentity() if err != nil { return commands.Context{}, err } p := ident.ID() conf := config.Config{ Identity: config.Identity{ PeerID: p.String(), }, } node, err := core.NewIPFSNode(context.Background(), core.Offline(&repo.Mock{ D: ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())), C: conf, })) return commands.Context{ Online: true, ConfigRoot: "/tmp/.mockipfsconfig", LoadConfig: func(path string) (*config.Config, error) { return &conf, nil }, ConstructNode: func() (*core.IpfsNode, error) { return node, nil }, }, nil }
func TestDuplicateSemantics(t *testing.T) { ctx := context.Background() dstore := dssync.MutexWrap(ds.NewMapDatastore()) bstore := blockstore.NewBlockstore(dstore) bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? p := NewPinner(dstore, dserv) a, _ := randNode() _, err := dserv.Add(a) if err != nil { t.Fatal(err) } // pin is recursively err = p.Pin(ctx, a, true) if err != nil { t.Fatal(err) } // pinning directly should fail err = p.Pin(ctx, a, false) if err == nil { t.Fatal("expected direct pin to fail") } // pinning recursively again should succeed err = p.Pin(ctx, a, true) if err != nil { t.Fatal(err) } }
func MocknetTestRepo(p peer.ID, h host.Host, conf testutil.LatencyConfig, routing core.RoutingOption) core.ConfigOption { return func(ctx context.Context) (*core.IpfsNode, error) { const kWriteCacheElems = 100 const alwaysSendToPeer = true dsDelay := delay.Fixed(conf.BlockstoreLatency) r := &repo.Mock{ D: ds2.CloserWrap(syncds.MutexWrap(ds2.WithDelay(datastore.NewMapDatastore(), dsDelay))), } ds := r.Datastore() n := &core.IpfsNode{ Peerstore: h.Peerstore(), Repo: r, PeerHost: h, Identity: p, } dhtt, err := routing(ctx, n.PeerHost, n.Repo.Datastore()) if err != nil { return nil, err } bsn := bsnet.NewFromIpfsHost(h, dhtt) bstore, err := blockstore.WriteCached(blockstore.NewBlockstore(ds), kWriteCacheElems) if err != nil { return nil, err } exch := bitswap.New(ctx, p, bsn, bstore, alwaysSendToPeer) n.Blockstore = bstore n.Exchange = exch n.Routing = dhtt return n, nil } }
func runFileCattingWorker(ctx context.Context, n *core.IpfsNode) error { conf, err := config.Init(ioutil.Discard, *nBitsForKeypair) if err != nil { return err } r := &repo.Mock{ D: ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())), C: *conf, } dummy, err := core.NewNode(ctx, &core.BuildCfg{ Repo: r, }) if err != nil { return err } errs := make(chan error) go func() { defer dummy.Close() var i int64 = 1 for { buf := new(bytes.Buffer) if err := random.WritePseudoRandomBytes(sizeOfIthFile(i), buf, *seed); err != nil { errs <- err } // add to a dummy node to discover the key k, err := coreunix.Add(dummy, bytes.NewReader(buf.Bytes())) if err != nil { errs <- err } e := elog.EventBegin(ctx, "cat", logging.LoggableF(func() map[string]interface{} { return map[string]interface{}{ "key": k, "localPeer": n.Identity, } })) if r, err := coreunix.Cat(ctx, n, k); err != nil { e.Done() log.Printf("failed to cat file. seed: %d #%d key: %s err: %s", *seed, i, k, err) } else { log.Println("found file", "seed", *seed, "#", i, "key", k, "size", unit.Information(sizeOfIthFile(i))) io.Copy(ioutil.Discard, r) e.Done() log.Println("catted file", "seed", *seed, "#", i, "key", k, "size", unit.Information(sizeOfIthFile(i))) i++ } time.Sleep(time.Second) } }() err = <-errs if err != nil { log.Fatal(err) } return nil }
func InitializeSupernodeNetwork( ctx context.Context, numServers, numClients int, conf testutil.LatencyConfig) ([]*core.IpfsNode, []*core.IpfsNode, error) { // create network mn, err := mocknet.FullMeshLinked(ctx, numServers+numClients) if err != nil { return nil, nil, err } mn.SetLinkDefaults(mocknet.LinkOptions{ Latency: conf.NetworkLatency, Bandwidth: math.MaxInt32, }) peers := mn.Peers() if len(peers) < numServers+numClients { return nil, nil, errors.New("test initialization error") } clientPeers, serverPeers := peers[0:numClients], peers[numClients:] routingDatastore := ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())) var servers []*core.IpfsNode for i := range iter.N(numServers) { p := serverPeers[i] bootstrap, err := core.NewIPFSNode(ctx, MocknetTestRepo(p, mn.Host(p), conf, corerouting.SupernodeServer(routingDatastore))) if err != nil { return nil, nil, err } servers = append(servers, bootstrap) } var bootstrapInfos []peer.PeerInfo for _, n := range servers { info := n.Peerstore.PeerInfo(n.PeerHost.ID()) bootstrapInfos = append(bootstrapInfos, info) } var clients []*core.IpfsNode for i := range iter.N(numClients) { p := clientPeers[i] n, err := core.NewIPFSNode(ctx, MocknetTestRepo(p, mn.Host(p), conf, corerouting.SupernodeClient(bootstrapInfos...))) if err != nil { return nil, nil, err } clients = append(clients, n) } bcfg := core.BootstrapConfigWithPeers(bootstrapInfos) for _, n := range clients { if err := n.Bootstrap(bcfg); err != nil { return nil, nil, err } } return servers, clients, nil }
func newEngine(ctx context.Context, idStr string) peerAndEngine { return peerAndEngine{ Peer: peer.ID(idStr), //Strategy: New(true), Engine: NewEngine(ctx, blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))), } }
func TestReturnsErrorWhenSizeNegative(t *testing.T) { bs := NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore())) _, err := WriteCached(bs, -1) if err != nil { return } t.Fail() }
func Mock(t testing.TB) dag.DAGService { bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) bserv, err := bsrv.New(bstore, offline.Exchange(bstore)) if err != nil { t.Fatal(err) } return dag.NewDAGService(bserv) }
func getMockDagServAndBstore(t testing.TB) (mdag.DAGService, blockstore.Blockstore, pin.ManualPinner) { dstore := ds.NewMapDatastore() tsds := sync.MutexWrap(dstore) bstore := blockstore.NewBlockstore(tsds) bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) return dserv, bstore, pin.NewPinner(tsds, dserv).GetManual() }
// NewPeerstore creates a threadsafe collection of peers. func NewPeerstore() Peerstore { return &peerstore{ keybook: *newKeybook(), metrics: *(NewMetrics()).(*metrics), AddrManager: AddrManager{}, ds: dssync.MutexWrap(ds.NewMapDatastore()), } }
func getDagserv(t *testing.T) merkledag.DAGService { db := dssync.MutexWrap(ds.NewMapDatastore()) bs := bstore.NewBlockstore(db) blockserv, err := bserv.New(bs, offline.Exchange(bs)) if err != nil { t.Fatal(err) } return merkledag.NewDAGService(blockserv) }
// NewMockNode constructs an IpfsNode for use in tests. func NewMockNode() (*core.IpfsNode, error) { ctx := context.Background() // Generate Identity ident, err := testutil.RandIdentity() if err != nil { return nil, err } p := ident.ID() c := config.Config{ Identity: config.Identity{ PeerID: p.String(), }, } nd, err := core.Offline(&repo.Mock{ C: c, D: ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())), })(ctx) if err != nil { return nil, err } nd.PrivateKey = ident.PrivateKey() nd.Peerstore = peer.NewPeerstore() nd.Peerstore.AddPrivKey(p, ident.PrivateKey()) nd.Peerstore.AddPubKey(p, ident.PublicKey()) nd.Identity = p nd.PeerHost, err = mocknet.New(nd.Context()).AddPeer(ident.PrivateKey(), ident.Address()) // effectively offline if err != nil { return nil, err } // Routing nd.Routing = offrt.NewOfflineRouter(nd.Repo.Datastore(), nd.PrivateKey) // Bitswap bstore := blockstore.NewBlockstore(nd.Repo.Datastore()) bserv, err := blockservice.New(bstore, offline.Exchange(bstore)) if err != nil { return nil, err } nd.DAG = mdag.NewDAGService(bserv) nd.Pinning = pin.NewPinner(nd.Repo.Datastore(), nd.DAG) // Namespace resolver nd.Namesys = nsys.NewNameSystem(nd.Routing) // Path resolver nd.Resolver = &path.Resolver{DAG: nd.DAG} return nd, nil }
func TestRecurivePathResolution(t *testing.T) { ctx := context.Background() dstore := sync.MutexWrap(datastore.NewMapDatastore()) bstore := blockstore.NewBlockstore(dstore) bserv, err := blockservice.New(bstore, offline.Exchange(bstore)) if err != nil { t.Fatal(err) } dagService := merkledag.NewDAGService(bserv) a, _ := randNode() b, _ := randNode() c, cKey := randNode() err = b.AddNodeLink("grandchild", c) if err != nil { t.Fatal(err) } err = a.AddNodeLink("child", b) if err != nil { t.Fatal(err) } err = dagService.AddRecursive(a) if err != nil { t.Fatal(err) } aKey, err := a.Key() if err != nil { t.Fatal(err) } segments := []string{aKey.String(), "child", "grandchild"} p, err := path.FromSegments("/ipfs/", segments...) if err != nil { t.Fatal(err) } resolver := &path.Resolver{DAG: dagService} node, err := resolver.ResolvePath(ctx, p) if err != nil { t.Fatal(err) } key, err := node.Key() if err != nil { t.Fatal(err) } if key.String() != cKey.String() { t.Fatal(fmt.Errorf( "recursive path resolution failed for %s: %s != %s", p.String(), key.String(), cKey.String())) } }
func (rs *mocknetserver) ClientWithDatastore(ctx context.Context, p testutil.Identity, ds ds.Datastore) Client { // FIXME AddPeer doesn't appear to be idempotent host, err := rs.mn.AddPeer(p.PrivateKey(), p.Address()) if err != nil { panic("FIXME") } return dht.NewDHT(ctx, host, sync.MutexWrap(ds)) }
func TestGetWhenKeyNotPresent(t *testing.T) { bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) _, err := bs.Get(key.Key("not present")) if err != nil { t.Log("As expected, block is not present") return } t.Fail() }
func InitializeSupernodeNetwork( ctx context.Context, numServers, numClients int, conf testutil.LatencyConfig) ([]*core.IpfsNode, []*core.IpfsNode, error) { // create network mn := mocknet.New(ctx) mn.SetLinkDefaults(mocknet.LinkOptions{ Latency: conf.NetworkLatency, Bandwidth: math.MaxInt32, }) routingDatastore := ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())) var servers []*core.IpfsNode for i := 0; i < numServers; i++ { bootstrap, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), Routing: corerouting.SupernodeServer(routingDatastore), }) if err != nil { return nil, nil, err } servers = append(servers, bootstrap) } var bootstrapInfos []peer.PeerInfo for _, n := range servers { info := n.Peerstore.PeerInfo(n.PeerHost.ID()) bootstrapInfos = append(bootstrapInfos, info) } var clients []*core.IpfsNode for i := 0; i < numClients; i++ { n, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), Routing: corerouting.SupernodeClient(bootstrapInfos...), }) if err != nil { return nil, nil, err } clients = append(clients, n) } mn.LinkAll() bcfg := core.BootstrapConfigWithPeers(bootstrapInfos) for _, n := range clients { if err := n.Bootstrap(bcfg); err != nil { return nil, nil, err } } return servers, clients, nil }
func getDagservAndPinner(t *testing.T) dagservAndPinner { db := dssync.MutexWrap(ds.NewMapDatastore()) bs := bstore.NewBlockstore(db) blockserv := bserv.New(bs, offline.Exchange(bs)) dserv := NewDAGService(blockserv) mpin := pin.NewPinner(db, dserv).GetManual() return dagservAndPinner{ ds: dserv, mp: mpin, } }
// session creates a test bitswap session. // // NB: It's easy make mistakes by providing the same peer ID to two different // sessions. To safeguard, use the SessionGenerator to generate sessions. It's // just a much better idea. func session(ctx context.Context, net tn.Network, p testutil.Identity) Instance { bsdelay := delay.Fixed(0) const writeCacheElems = 100 adapter := net.Adapter(p) dstore := ds_sync.MutexWrap(datastore2.WithDelay(ds.NewMapDatastore(), bsdelay)) bstore, err := blockstore.WriteCached(blockstore.NewBlockstore(ds_sync.MutexWrap(dstore)), writeCacheElems) if err != nil { panic(err.Error()) // FIXME perhaps change signature and return error. } const alwaysSendToPeer = true bs := New(ctx, p.ID(), adapter, bstore, alwaysSendToPeer).(*Bitswap) return Instance{ Peer: p.ID(), Exchange: bs, blockstore: bstore, blockstoreDelay: bsdelay, } }
func setupDHT(ctx context.Context, t *testing.T) *IpfsDHT { h := netutil.GenHostSwarm(t, ctx) dss := dssync.MutexWrap(ds.NewMapDatastore()) d := NewDHT(ctx, h, dss) d.Validator["v"] = &record.ValidChecker{ Func: func(key.Key, []byte) error { return nil }, Sign: false, } return d }
func TestValueTypeMismatch(t *testing.T) { block := blocks.NewBlock([]byte("some data")) datastore := ds.NewMapDatastore() k := BlockPrefix.Child(block.Key().DsKey()) datastore.Put(k, "data that isn't a block!") blockstore := NewBlockstore(ds_sync.MutexWrap(datastore)) _, err := blockstore.Get(block.Key()) if err != ValueTypeMismatch { t.Fatal(err) } }
func TestElideDuplicateWrite(t *testing.T) { cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()} bs := NewBlockstore(syncds.MutexWrap(cd)) cachedbs, err := WriteCached(bs, 1) if err != nil { t.Fatal(err) } b1 := blocks.NewBlock([]byte("foo")) cachedbs.Put(b1) cd.SetFunc(func() { t.Fatal("write hit the datastore") }) cachedbs.Put(b1) }
func TestOutboxClosedWhenEngineClosed(t *testing.T) { t.SkipNow() // TODO implement *Engine.Close e := NewEngine(context.Background(), blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))) var wg sync.WaitGroup wg.Add(1) go func() { for nextEnvelope := range e.Outbox() { <-nextEnvelope } wg.Done() }() // e.Close() wg.Wait() if _, ok := <-e.Outbox(); ok { t.Fatal("channel should be closed") } }
func newBlockStoreWithKeys(t *testing.T, d ds.Datastore, N int) (Blockstore, []key.Key) { if d == nil { d = ds.NewMapDatastore() } bs := NewBlockstore(ds_sync.MutexWrap(d)) keys := make([]key.Key, N) for i := 0; i < N; i++ { block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i))) err := bs.Put(block) if err != nil { t.Fatal(err) } keys[i] = block.Key() } return bs, keys }
func TestPutThenGetBlock(t *testing.T) { bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) block := blocks.NewBlock([]byte("some data")) err := bs.Put(block) if err != nil { t.Fatal(err) } blockFromBlockstore, err := bs.Get(block.Key()) if err != nil { t.Fatal(err) } if !bytes.Equal(block.Data, blockFromBlockstore.Data) { t.Fail() } }
func TestBlocks(t *testing.T) { bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) bs, err := New(bstore, offline.Exchange(bstore)) if err != nil { t.Error("failed to construct block service", err) return } defer bs.Close() b := blocks.NewBlock([]byte("beep boop")) h := u.Hash([]byte("beep boop")) if !bytes.Equal(b.Multihash, h) { t.Error("Block Multihash and data multihash not equal") } if b.Key() != key.Key(h) { t.Error("Block key and data multihash key not equal") } k, err := bs.AddBlock(b) if err != nil { t.Error("failed to add block to BlockService", err) return } if k != b.Key() { t.Error("returned key is not equal to block key", err) } ctx, _ := context.WithTimeout(context.TODO(), time.Second*5) b2, err := bs.GetBlock(ctx, b.Key()) if err != nil { t.Error("failed to retrieve block from BlockService", err) return } if b.Key() != b2.Key() { t.Error("Block keys not equal.") } if !bytes.Equal(b.Data, b2.Data) { t.Error("Block data is not equal.") } }
func TestPartnerWantsThenCancels(t *testing.T) { numRounds := 10 if testing.Short() { numRounds = 1 } alphabet := strings.Split("abcdefghijklmnopqrstuvwxyz", "") vowels := strings.Split("aeiou", "") type testCase [][]string testcases := []testCase{ { alphabet, vowels, }, { alphabet, stringsComplement(alphabet, vowels), }, } bs := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) for _, letter := range alphabet { block := blocks.NewBlock([]byte(letter)) if err := bs.Put(block); err != nil { t.Fatal(err) } } for i := 0; i < numRounds; i++ { for _, testcase := range testcases { set := testcase[0] cancels := testcase[1] keeps := stringsComplement(set, cancels) e := NewEngine(context.Background(), bs) partner := testutil.RandPeerIDFatal(t) partnerWants(e, set, partner) partnerCancels(e, cancels, partner) if err := checkHandledInOrder(t, e, keeps); err != nil { t.Logf("run #%d of %d", i, numRounds) t.Fatal(err) } } } }
func (nb *NodeBuilder) Build(ctx context.Context) (*IpfsNode, error) { if nb.built { return nil, ErrAlreadyBuilt } nb.built = true if nb.repo == nil { var d ds.Datastore d = ds.NewMapDatastore() if nb.nilrepo { d = ds.NewNullDatastore() } r, err := defaultRepo(dsync.MutexWrap(d)) if err != nil { return nil, err } nb.repo = r } conf := standardWithRouting(nb.repo, nb.online, nb.routing, nb.peerhost) return NewIPFSNode(ctx, conf) }
func setup() *mock { m := &mock{} mp := ds.NewMapDatastore() ts := dssync.MutexWrap(mp) cb1 := dscb.Wrap(ts, func() { m.Lock() m.inside++ m.Unlock() <-time.After(20 * time.Millisecond) }) cd := Wrap(cb1) cb2 := dscb.Wrap(cd, func() { m.Lock() m.outside++ m.Unlock() }) m.ds = cb2 return m }