func TestDoubleGet(t *testing.T) { net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) sg := NewTestSessionGenerator(net) defer sg.Close() bg := blocksutil.NewBlockGenerator() t.Log("Test a one node trying to get one block from another") instances := sg.Instances(2) blocks := bg.Blocks(1) ctx1, cancel1 := context.WithCancel(context.Background()) blkch1, err := instances[1].Exchange.GetBlocks(ctx1, []key.Key{blocks[0].Key()}) if err != nil { t.Fatal(err) } ctx2, cancel2 := context.WithCancel(context.Background()) defer cancel2() blkch2, err := instances[1].Exchange.GetBlocks(ctx2, []key.Key{blocks[0].Key()}) if err != nil { t.Fatal(err) } // ensure both requests make it into the wantlist at the same time time.Sleep(time.Millisecond * 100) cancel1() _, ok := <-blkch1 if ok { t.Fatal("expected channel to be closed") } err = instances[0].Exchange.HasBlock(blocks[0]) if err != nil { t.Fatal(err) } blk, ok := <-blkch2 if !ok { t.Fatal("expected to get the block here") } t.Log(blk) for _, inst := range instances { err := inst.Exchange.Close() if err != nil { t.Fatal(err) } } }
func RunSupernodePutRecordGetRecord(conf testutil.LatencyConfig) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() servers, clients, err := InitializeSupernodeNetwork(ctx, 2, 2, conf) if err != nil { return err } for _, n := range append(servers, clients...) { defer n.Close() } putter := clients[0] getter := clients[1] k := key.Key("key") note := []byte("a note from putter") if err := putter.Routing.PutValue(ctx, k, note); err != nil { return fmt.Errorf("failed to put value: %s", err) } received, err := getter.Routing.GetValue(ctx, k) if err != nil { return fmt.Errorf("failed to get value: %s", err) } if 0 != bytes.Compare(note, received) { return errors.New("record doesn't match") } cancel() return nil }
func (bs *Bitswap) rebroadcastWorker(parent context.Context) { ctx, cancel := context.WithCancel(parent) defer cancel() broadcastSignal := time.NewTicker(rebroadcastDelay.Get()) defer broadcastSignal.Stop() tick := time.NewTicker(10 * time.Second) defer tick.Stop() for { log.Event(ctx, "Bitswap.Rebroadcast.idle") select { case <-tick.C: n := bs.wm.wl.Len() if n > 0 { log.Debug(n, "keys in bitswap wantlist") } case <-broadcastSignal.C: // resend unfulfilled wantlist keys log.Event(ctx, "Bitswap.Rebroadcast.active") entries := bs.wm.wl.Entries() if len(entries) > 0 { bs.connectToProviders(ctx, entries) } case <-parent.Done(): return } } }
func TestWritePostCancel(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) piper, pipew := io.Pipe() w := NewWriter(ctx, pipew) buf := []byte("abcdefghij") buf2 := make([]byte, 10) done := make(chan ioret) go func() { n, err := w.Write(buf) done <- ioret{n, err} }() piper.Read(buf2) select { case ret := <-done: if ret.n != 10 { t.Error("ret.n should be 10", ret.n) } if ret.err != nil { t.Error("ret.err should be nil", ret.err) } if string(buf2) != "abcdefghij" { t.Error("write contents differ") } case <-time.After(20 * time.Millisecond): t.Fatal("failed to write") } go func() { n, err := w.Write(buf) done <- ioret{n, err} }() cancel() select { case ret := <-done: if ret.n != 0 { t.Error("ret.n should be 0", ret.n) } if ret.err == nil { t.Error("ret.err should be ctx error", ret.err) } case <-time.After(20 * time.Millisecond): t.Fatal("failed to stop writing after cancel") } copy(buf, []byte("aaaaaaaaaa")) piper.Read(buf2) if string(buf2) == "aaaaaaaaaa" { t.Error("buffer was read from after ctx cancel") } else if string(buf2) != "abcdefghij" { t.Error("write contents differ from expected") } }
func TestBasic(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ds, rt := setupRoot(ctx, t) rootdir := rt.GetValue().(*Directory) // test making a basic dir _, err := rootdir.Mkdir("a") if err != nil { t.Fatal(err) } path := "a/b/c/d/e/f/g" d := mkdirP(t, rootdir, path) fi := getRandFile(t, ds, 1000) // test inserting that file err = d.AddChild("afile", fi) if err != nil { t.Fatal(err) } err = assertFileAtPath(ds, rootdir, fi, "a/b/c/d/e/f/g/afile") if err != nil { t.Fatal(err) } }
func (i *cmdInvocation) SetupInterruptHandler(ctx context.Context) (io.Closer, context.Context) { intrh := NewIntrHandler() ctx, cancelFunc := context.WithCancel(ctx) handlerFunc := func(count int, ih *IntrHandler) { switch count { case 1: fmt.Println() // Prevent un-terminated ^C character in terminal ih.wg.Add(1) go func() { defer ih.wg.Done() cancelFunc() }() default: fmt.Println("Received another interrupt before graceful shutdown, terminating...") os.Exit(-1) } } intrh.Handle(handlerFunc, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) return intrh, ctx }
// WithDeadlineFraction returns a Context with a fraction of the // original context's timeout. This is useful in sequential pipelines // of work, where one might try options and fall back to others // depending on the time available, or failure to respond. For example: // // // getPicture returns a picture from our encrypted database // // we have a pipeline of multiple steps. we need to: // // - get the data from a database // // - decrypt it // // - apply many transforms // // // // we **know** that each step takes increasingly more time. // // The transforms are much more expensive than decryption, and // // decryption is more expensive than the database lookup. // // If our database takes too long (i.e. >0.2 of available time), // // there's no use in continuing. // func getPicture(ctx context.Context, key string) ([]byte, error) { // // fractional timeout contexts to the rescue! // // // try the database with 0.2 of remaining time. // ctx1, _ := ctxext.WithDeadlineFraction(ctx, 0.2) // val, err := db.Get(ctx1, key) // if err != nil { // return nil, err // } // // // try decryption with 0.3 of remaining time. // ctx2, _ := ctxext.WithDeadlineFraction(ctx, 0.3) // if val, err = decryptor.Decrypt(ctx2, val); err != nil { // return nil, err // } // // // try transforms with all remaining time. hopefully it's enough! // return transformer.Transform(ctx, val) // } // // func WithDeadlineFraction(ctx context.Context, fraction float64) ( context.Context, context.CancelFunc) { d, found := ctx.Deadline() if !found { // no deadline return context.WithCancel(ctx) } left := d.Sub(time.Now()) if left < 0 { // already passed... return context.WithCancel(ctx) } left = time.Duration(float64(left) * fraction) return context.WithTimeout(ctx, left) }
func TestConsistentAccounting(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() sender := newEngine(ctx, "Ernie") receiver := newEngine(ctx, "Bert") // Send messages from Ernie to Bert for i := 0; i < 1000; i++ { m := message.New(false) content := []string{"this", "is", "message", "i"} m.AddBlock(blocks.NewBlock([]byte(strings.Join(content, " ")))) sender.Engine.MessageSent(receiver.Peer, m) receiver.Engine.MessageReceived(sender.Peer, m) } // Ensure sender records the change if sender.Engine.numBytesSentTo(receiver.Peer) == 0 { t.Fatal("Sent bytes were not recorded") } // Ensure sender and receiver have the same values if sender.Engine.numBytesSentTo(receiver.Peer) != receiver.Engine.numBytesReceivedFrom(sender.Peer) { t.Fatal("Inconsistent book-keeping. Strategies don't agree") } // Ensure sender didn't record receving anything. And that the receiver // didn't record sending anything if receiver.Engine.numBytesSentTo(sender.Peer) != 0 || sender.Engine.numBytesReceivedFrom(receiver.Peer) != 0 { t.Fatal("Bert didn't send bytes to Ernie") } }
func initializeIpnsKeyspace(repoRoot string, privKeyBytes []byte) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() r, err := fsrepo.Open(repoRoot) if err != nil { // NB: repo is owned by the node return err } cfg, err := r.Config() if err != nil { log.Error(err) return err } identity, err := ipfs.IdentityFromKey(privKeyBytes) if err != nil { return err } cfg.Identity = identity nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r}) if err != nil { return err } defer nd.Close() err = nd.SetupOfflineRouting() if err != nil { return err } return namesys.InitializeKeyspace(ctx, nd.DAG, nd.Namesys, nd.Pinning, nd.PrivateKey) }
func (bs *Bitswap) connectToProviders(ctx context.Context, entries []wantlist.Entry) { ctx, cancel := context.WithCancel(ctx) defer cancel() // Get providers for all entries in wantlist (could take a while) wg := sync.WaitGroup{} for _, e := range entries { wg.Add(1) go func(k key.Key) { defer wg.Done() child, cancel := context.WithTimeout(ctx, providerRequestTimeout) defer cancel() providers := bs.network.FindProvidersAsync(child, k, maxProvidersPerRequest) for prov := range providers { go func(p peer.ID) { bs.network.ConnectTo(ctx, p) }(prov) } }(e.Key) } wg.Wait() // make sure all our children do finish. }
func Unpin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]key.Key, error) { var unpinned []key.Key for _, p := range paths { p, err := path.ParsePath(p) if err != nil { return nil, err } k, err := core.ResolveToKey(ctx, n, p) if err != nil { return nil, err } ctx, cancel := context.WithCancel(ctx) defer cancel() err = n.Pinning.Unpin(ctx, k, recursive) if err != nil { return nil, err } unpinned = append(unpinned, k) } err := n.Pinning.Flush() if err != nil { return nil, err } return unpinned, nil }
func Pin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]key.Key, error) { dagnodes := make([]*merkledag.Node, 0) for _, fpath := range paths { dagnode, err := core.Resolve(ctx, n, path.Path(fpath)) if err != nil { return nil, fmt.Errorf("pin: %s", err) } dagnodes = append(dagnodes, dagnode) } var out []key.Key for _, dagnode := range dagnodes { k, err := dagnode.Key() if err != nil { return nil, err } ctx, cancel := context.WithCancel(ctx) defer cancel() err = n.Pinning.Pin(ctx, dagnode, recursive) if err != nil { return nil, fmt.Errorf("pin: %s", err) } out = append(out, k) } err := n.Pinning.Flush() if err != nil { return nil, err } return out, nil }
func BenchmarkDagmodWrite(b *testing.B) { b.StopTimer() dserv := getMockDagServ(b) _, n := getNode(b, dserv, 0) ctx, cancel := context.WithCancel(context.Background()) defer cancel() wrsize := 4096 dagmod, err := NewDagModifier(ctx, n, dserv, sizeSplitterGen(512)) if err != nil { b.Fatal(err) } buf := make([]byte, b.N*wrsize) u.NewTimeSeededRand().Read(buf) b.StartTimer() b.SetBytes(int64(wrsize)) for i := 0; i < b.N; i++ { n, err := dagmod.Write(buf[i*wrsize : (i+1)*wrsize]) if err != nil { b.Fatal(err) } if n != wrsize { b.Fatal("Wrote bad size") } } }
func addDefaultAssets(out io.Writer, repoRoot string) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() r, err := fsrepo.Open(repoRoot) if err != nil { // NB: repo is owned by the node return err } nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r}) if err != nil { return err } defer nd.Close() dkey, err := assets.SeedInitDocs(nd) if err != nil { return fmt.Errorf("init: seeding init docs failed: %s", err) } log.Debugf("init: seeded init docs %s", dkey) if _, err = fmt.Fprintf(out, "to get started, enter:\n"); err != nil { return err } _, err = fmt.Fprintf(out, "\n\tipfs cat /ipfs/%s/readme\n\n", dkey) return err }
func (dm *DagModifier) readPrep() error { err := dm.Sync() if err != nil { return err } if dm.read == nil { ctx, cancel := context.WithCancel(dm.ctx) dr, err := uio.NewDagReader(ctx, dm.curNode, dm.dagserv) if err != nil { return err } i, err := dr.Seek(int64(dm.curWrOff), os.SEEK_SET) if err != nil { return err } if i != int64(dm.curWrOff) { return ErrSeekFail } dm.readCancel = cancel dm.read = dr } return nil }
// InitializeKeyspace sets the ipns record for the given key to // point to an empty directory. func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error { emptyDir := &mdag.Node{Data: ft.FolderPBData()} nodek, err := n.DAG.Add(emptyDir) if err != nil { return err } ctx, cancel := context.WithCancel(n.Context()) defer cancel() err = n.Pinning.Pin(ctx, emptyDir, false) if err != nil { return err } err = n.Pinning.Flush() if err != nil { return err } pub := nsys.NewRoutingPublisher(n.Routing, n.Repo.Datastore()) if err := pub.Publish(ctx, key, path.FromKey(nodek)); err != nil { return err } return nil }
func (rp *Republisher) republishEntries(p goprocess.Process) error { ctx, cancel := context.WithCancel(gpctx.OnClosingContext(p)) defer cancel() for id, _ := range rp.entries { log.Debugf("republishing ipns entry for %s", id) priv := rp.ps.PrivKey(id) // Look for it locally only _, ipnskey := namesys.IpnsKeysForID(id) p, seq, err := rp.getLastVal(ipnskey) if err != nil { if err == errNoEntry { continue } return err } // update record with same sequence number eol := time.Now().Add(rp.RecordLifetime) err = namesys.PutRecordToRouting(ctx, priv, p, seq, eol, rp.r, id) if err != nil { return err } } return nil }
// Get retrieves a node from the dagService, fetching the block in the BlockService func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) { if k == "" { return nil, ErrNotFound } if n == nil { return nil, fmt.Errorf("dagService is nil") } ctx, cancel := context.WithCancel(ctx) defer cancel() b, err := n.Blocks.GetBlock(ctx, k) if err != nil { if err == bserv.ErrNotFound { return nil, ErrNotFound } return nil, fmt.Errorf("Failed to get block for %s: %v", k.B58String(), err) } res, err := DecodeProtobuf(b.Data()) if err != nil { if strings.Contains(err.Error(), "Unmarshal failed") { return nil, fmt.Errorf("The block referred to by '%s' was not a valid merkledag node", k) } return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err) } res.cached = k.ToMultihash() return res, nil }
func TestDoesNotDeadLockIfContextCancelledBeforePublish(t *testing.T) { g := blocksutil.NewBlockGenerator() ctx, cancel := context.WithCancel(context.Background()) n := New() defer n.Shutdown() t.Log("generate a large number of blocks. exceed default buffer") bs := g.Blocks(1000) ks := func() []key.Key { var keys []key.Key for _, b := range bs { keys = append(keys, b.Key()) } return keys }() _ = n.Subscribe(ctx, ks...) // ignore received channel t.Log("cancel context before any blocks published") cancel() for _, b := range bs { n.Publish(b) } t.Log("publishing the large number of blocks to the ignored channel must not deadlock") }
func TestReadPostCancel(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) piper, pipew := io.Pipe() r := NewReader(ctx, piper) buf := make([]byte, 10) done := make(chan ioret) go func() { n, err := r.Read(buf) done <- ioret{n, err} }() cancel() select { case ret := <-done: if ret.n != 0 { t.Error("ret.n should be 0", ret.n) } if ret.err == nil { t.Error("ret.err should be ctx error", ret.err) } case <-time.After(20 * time.Millisecond): t.Fatal("failed to stop reading after cancel") } pipew.Write([]byte("abcdefghij")) if !bytes.Equal(buf, make([]byte, len(buf))) { t.Fatal("buffer should have not been written to") } }
// WithProcessClosing returns a context.Context derived from ctx that // is cancelled as p is Closing (after: <-p.Closing()). It is simply: // // func WithProcessClosing(ctx context.Context, p goprocess.Process) context.Context { // ctx, cancel := context.WithCancel(ctx) // go func() { // <-p.Closing() // cancel() // }() // return ctx // } // func WithProcessClosing(ctx context.Context, p goprocess.Process) context.Context { ctx, cancel := context.WithCancel(ctx) go func() { <-p.Closing() cancel() }() return ctx }
// TODO does dht ensure won't receive self as a provider? probably not. func TestCanceledContext(t *testing.T) { rs := NewServer() k := key.Key("hello") // avoid leaking goroutine, without using the context to signal // (we want the goroutine to keep trying to publish on a // cancelled context until we've tested it doesnt do anything.) done := make(chan struct{}) defer func() { done <- struct{}{} }() t.Log("async'ly announce infinite stream of providers for key") i := 0 go func() { // infinite stream for { select { case <-done: t.Log("exiting async worker") return default: } pi, err := testutil.RandIdentity() if err != nil { t.Error(err) } err = rs.Client(pi).Provide(context.Background(), k) if err != nil { t.Error(err) } i++ } }() local := testutil.RandIdentityOrFatal(t) client := rs.Client(local) t.Log("warning: max is finite so this test is non-deterministic") t.Log("context cancellation could simply take lower priority") t.Log("and result in receiving the max number of results") max := 1000 t.Log("cancel the context before consuming") ctx, cancelFunc := context.WithCancel(context.Background()) cancelFunc() providers := client.FindProvidersAsync(ctx, k, max) numProvidersReturned := 0 for _ = range providers { numProvidersReturned++ } t.Log(numProvidersReturned) if numProvidersReturned == max { t.Fatal("Context cancel had no effect") } }
// GetNodes returns an array of 'NodeGetter' promises, with each corresponding // to the key with the same index as the passed in keys func GetNodes(ctx context.Context, ds DAGService, keys []key.Key) []NodeGetter { // Early out if no work to do if len(keys) == 0 { return nil } promises := make([]NodeGetter, len(keys)) for i := range keys { promises[i] = newNodePromise(ctx) } dedupedKeys := dedupeKeys(keys) go func() { ctx, cancel := context.WithCancel(ctx) defer cancel() nodechan := ds.GetMany(ctx, dedupedKeys) for count := 0; count < len(keys); { select { case opt, ok := <-nodechan: if !ok { for _, p := range promises { p.Fail(ErrNotFound) } return } if opt.Err != nil { for _, p := range promises { p.Fail(opt.Err) } return } nd := opt.Node k, err := nd.Key() if err != nil { log.Error("Failed to get node key: ", err) continue } is := FindLinks(keys, k, 0) for _, i := range is { count++ promises[i].Send(nd) } case <-ctx.Done(): return } } }() return promises }
// WARNING: this uses RandTestBogusIdentity DO NOT USE for NON TESTS! func NewTestSessionGenerator( net tn.Network) SessionGenerator { ctx, cancel := context.WithCancel(context.Background()) return SessionGenerator{ net: net, seq: 0, ctx: ctx, // TODO take ctx as param to Next, Instances cancel: cancel, } }
func EnumerateChildrenAsync(ctx context.Context, ds DAGService, root *Node, set key.KeySet) error { toprocess := make(chan []key.Key, 8) nodes := make(chan *NodeOption, 8) ctx, cancel := context.WithCancel(ctx) defer cancel() defer close(toprocess) go fetchNodes(ctx, ds, toprocess, nodes) nodes <- &NodeOption{Node: root} live := 1 for { select { case opt, ok := <-nodes: if !ok { return nil } if opt.Err != nil { return opt.Err } nd := opt.Node // a node has been fetched live-- var keys []key.Key for _, lnk := range nd.Links { k := key.Key(lnk.Hash) if !set.Has(k) { set.Add(k) live++ keys = append(keys, k) } } if live == 0 { return nil } if len(keys) > 0 { select { case toprocess <- keys: case <-ctx.Done(): return ctx.Err() } } case <-ctx.Done(): return ctx.Err() } } }
func verifyResolutionFails(nodes []*core.IpfsNode, key string) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() for _, n := range nodes { _, err := n.Namesys.Resolve(ctx, key) if err == nil { return errors.New("expected resolution to fail") } } return nil }
func TestDoReturnsContextErr(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) ch := make(chan struct{}) err := ContextDo(ctx, func() error { cancel() ch <- struct{}{} // won't return return nil }) if err != ctx.Err() { t.Fail() } }
// NewRepublisher creates a new Republisher object to republish the given root // using the given short and long time intervals func NewRepublisher(ctx context.Context, pf PubFunc, tshort, tlong time.Duration) *Republisher { ctx, cancel := context.WithCancel(ctx) return &Republisher{ TimeoutShort: tshort, TimeoutLong: tlong, Publish: make(chan struct{}, 1), pubfunc: pf, pubnowch: make(chan chan struct{}), ctx: ctx, cancel: cancel, } }
func TestDagModifierBasic(t *testing.T) { dserv := getMockDagServ(t) b, n := getNode(t, dserv, 50000) ctx, cancel := context.WithCancel(context.Background()) defer cancel() dagmod, err := NewDagModifier(ctx, n, dserv, sizeSplitterGen(512)) if err != nil { t.Fatal(err) } // Within zero block beg := uint64(15) length := uint64(60) t.Log("Testing mod within zero block") b = testModWrite(t, beg, length, b, dagmod) // Within bounds of existing file beg = 1000 length = 4000 t.Log("Testing mod within bounds of existing multiblock file.") b = testModWrite(t, beg, length, b, dagmod) // Extend bounds beg = 49500 length = 4000 t.Log("Testing mod that extends file.") b = testModWrite(t, beg, length, b, dagmod) // "Append" beg = uint64(len(b)) length = 3000 t.Log("Testing pure append") b = testModWrite(t, beg, length, b, dagmod) // Verify reported length node, err := dagmod.GetNode() if err != nil { t.Fatal(err) } size, err := ft.DataSize(node.Data()) if err != nil { t.Fatal(err) } expected := uint64(50000 + 3500 + 3000) if size != expected { t.Fatalf("Final reported size is incorrect [%d != %d]", size, expected) } }
func TestMultiWrite(t *testing.T) { dserv := getMockDagServ(t) _, n := getNode(t, dserv, 0) ctx, cancel := context.WithCancel(context.Background()) defer cancel() dagmod, err := NewDagModifier(ctx, n, dserv, sizeSplitterGen(512)) if err != nil { t.Fatal(err) } data := make([]byte, 4000) u.NewTimeSeededRand().Read(data) for i := 0; i < len(data); i++ { n, err := dagmod.WriteAt(data[i:i+1], int64(i)) if err != nil { t.Fatal(err) } if n != 1 { t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)") } size, err := dagmod.Size() if err != nil { t.Fatal(err) } if size != int64(i+1) { t.Fatal("Size was reported incorrectly") } } nd, err := dagmod.GetNode() if err != nil { t.Fatal(err) } read, err := uio.NewDagReader(context.Background(), nd, dserv) if err != nil { t.Fatal(err) } rbuf, err := ioutil.ReadAll(read) if err != nil { t.Fatal(err) } err = arrComp(rbuf, data) if err != nil { t.Fatal(err) } }