func TestReadPostCancel(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
piper, pipew := io.Pipe()
r := NewReader(ctx, piper)
buf := make([]byte, 10)
done := make(chan ioret)
go func() {
n, err := r.Read(buf)
done <- ioret{n, err}
}()
cancel()
select {
case ret := <-done:
if ret.n != 0 {
t.Error("ret.n should be 0", ret.n)
}
if ret.err == nil {
t.Error("ret.err should be ctx error", ret.err)
}
case <-time.After(20 * time.Millisecond):
t.Fatal("failed to stop reading after cancel")
}
pipew.Write([]byte("abcdefghij"))
if !bytes.Equal(buf, make([]byte, len(buf))) {
t.Fatal("buffer should have not been written to")
}
}
func TestWritePostCancel(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
piper, pipew := io.Pipe()
w := NewWriter(ctx, pipew)
buf := []byte("abcdefghij")
buf2 := make([]byte, 10)
done := make(chan ioret)
go func() {
n, err := w.Write(buf)
done <- ioret{n, err}
}()
piper.Read(buf2)
select {
case ret := <-done:
if ret.n != 10 {
t.Error("ret.n should be 10", ret.n)
}
if ret.err != nil {
t.Error("ret.err should be nil", ret.err)
}
if string(buf2) != "abcdefghij" {
t.Error("write contents differ")
}
case <-time.After(20 * time.Millisecond):
t.Fatal("failed to write")
}
go func() {
n, err := w.Write(buf)
done <- ioret{n, err}
}()
cancel()
select {
case ret := <-done:
if ret.n != 0 {
t.Error("ret.n should be 0", ret.n)
}
if ret.err == nil {
t.Error("ret.err should be ctx error", ret.err)
}
case <-time.After(20 * time.Millisecond):
t.Fatal("failed to stop writing after cancel")
}
copy(buf, []byte("aaaaaaaaaa"))
piper.Read(buf2)
if string(buf2) == "aaaaaaaaaa" {
t.Error("buffer was read from after ctx cancel")
} else if string(buf2) != "abcdefghij" {
t.Error("write contents differ from expected")
}
}
func TestConsistentAccounting(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sender := newEngine(ctx, "Ernie")
receiver := newEngine(ctx, "Bert")
// Send messages from Ernie to Bert
for i := 0; i < 1000; i++ {
m := message.New(false)
content := []string{"this", "is", "message", "i"}
m.AddBlock(blocks.NewBlock([]byte(strings.Join(content, " "))))
sender.Engine.MessageSent(receiver.Peer, m)
receiver.Engine.MessageReceived(sender.Peer, m)
}
// Ensure sender records the change
if sender.Engine.numBytesSentTo(receiver.Peer) == 0 {
t.Fatal("Sent bytes were not recorded")
}
// Ensure sender and receiver have the same values
if sender.Engine.numBytesSentTo(receiver.Peer) != receiver.Engine.numBytesReceivedFrom(sender.Peer) {
t.Fatal("Inconsistent book-keeping. Strategies don't agree")
}
// Ensure sender didn't record receving anything. And that the receiver
// didn't record sending anything
if receiver.Engine.numBytesSentTo(sender.Peer) != 0 || sender.Engine.numBytesReceivedFrom(receiver.Peer) != 0 {
t.Fatal("Bert didn't send bytes to Ernie")
}
}
func BenchmarkDagmodWrite(b *testing.B) {
b.StopTimer()
dserv, pins := getMockDagServ(b)
_, n := getNode(b, dserv, 0, pins)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
wrsize := 4096
dagmod, err := NewDagModifier(ctx, n, dserv, pins, sizeSplitterGen(512))
if err != nil {
b.Fatal(err)
}
buf := make([]byte, b.N*wrsize)
u.NewTimeSeededRand().Read(buf)
b.StartTimer()
b.SetBytes(int64(wrsize))
for i := 0; i < b.N; i++ {
n, err := dagmod.Write(buf[i*wrsize : (i+1)*wrsize])
if err != nil {
b.Fatal(err)
}
if n != wrsize {
b.Fatal("Wrote bad size")
}
}
}
func TestDoesNotDeadLockIfContextCancelledBeforePublish(t *testing.T) {
g := blocksutil.NewBlockGenerator()
ctx, cancel := context.WithCancel(context.Background())
n := New()
defer n.Shutdown()
t.Log("generate a large number of blocks. exceed default buffer")
bs := g.Blocks(1000)
ks := func() []key.Key {
var keys []key.Key
for _, b := range bs {
keys = append(keys, b.Key())
}
return keys
}()
_ = n.Subscribe(ctx, ks...) // ignore received channel
t.Log("cancel context before any blocks published")
cancel()
for _, b := range bs {
n.Publish(b)
}
t.Log("publishing the large number of blocks to the ignored channel must not deadlock")
}
// newKeyRoot creates a new KeyRoot for the given key, and starts up a republisher routine
// for it
func (fs *Filesystem) newKeyRoot(parent context.Context, k ci.PrivKey) (*KeyRoot, error) {
hash, err := k.GetPublic().Hash()
if err != nil {
return nil, err
}
name := "/ipns/" + key.Key(hash).String()
root := new(KeyRoot)
root.key = k
root.fs = fs
root.name = name
ctx, cancel := context.WithCancel(parent)
defer cancel()
pointsTo, err := fs.nsys.Resolve(ctx, name)
if err != nil {
err = namesys.InitializeKeyspace(ctx, fs.dserv, fs.nsys, fs.pins, k)
if err != nil {
return nil, err
}
pointsTo, err = fs.nsys.Resolve(ctx, name)
if err != nil {
return nil, err
}
}
mnode, err := fs.resolver.ResolvePath(ctx, pointsTo)
if err != nil {
log.Errorf("Failed to retrieve value '%s' for ipns entry: %s\n", pointsTo, err)
return nil, err
}
root.node = mnode
root.repub = NewRepublisher(root, time.Millisecond*300, time.Second*3)
go root.repub.Run(parent)
pbn, err := ft.FromBytes(mnode.Data)
if err != nil {
log.Error("IPNS pointer was not unixfs node")
return nil, err
}
switch pbn.GetType() {
case ft.TDirectory:
root.val = NewDirectory(ctx, pointsTo.String(), mnode, root, fs)
case ft.TFile, ft.TMetadata, ft.TRaw:
fi, err := NewFile(pointsTo.String(), mnode, root, fs)
if err != nil {
return nil, err
}
root.val = fi
default:
panic("unrecognized! (NYI)")
}
return root, nil
}
func RunSupernodePutRecordGetRecord(conf testutil.LatencyConfig) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
servers, clients, err := InitializeSupernodeNetwork(ctx, 2, 2, conf)
if err != nil {
return err
}
for _, n := range append(servers, clients...) {
defer n.Close()
}
putter := clients[0]
getter := clients[1]
k := key.Key("key")
note := []byte("a note from putter")
if err := putter.Routing.PutValue(ctx, k, note); err != nil {
return fmt.Errorf("failed to put value: %s", err)
}
received, err := getter.Routing.GetValue(ctx, k)
if err != nil {
return fmt.Errorf("failed to get value: %s", err)
}
if 0 != bytes.Compare(note, received) {
return errors.New("record doesn't match")
}
return nil
}
func (bs *Bitswap) rebroadcastWorker(parent context.Context) {
ctx, cancel := context.WithCancel(parent)
defer cancel()
broadcastSignal := time.NewTicker(rebroadcastDelay.Get())
defer broadcastSignal.Stop()
tick := time.NewTicker(10 * time.Second)
defer tick.Stop()
for {
log.Event(ctx, "Bitswap.Rebroadcast.idle")
select {
case <-tick.C:
n := bs.wm.wl.Len()
if n > 0 {
log.Debug(n, "keys in bitswap wantlist")
}
case <-broadcastSignal.C: // resend unfulfilled wantlist keys
log.Event(ctx, "Bitswap.Rebroadcast.active")
entries := bs.wm.wl.Entries()
if len(entries) > 0 {
bs.connectToProviders(ctx, entries)
}
case <-parent.Done():
return
}
}
}
func Pin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]key.Key, error) {
dagnodes := make([]*merkledag.Node, 0)
for _, fpath := range paths {
dagnode, err := core.Resolve(ctx, n, path.Path(fpath))
if err != nil {
return nil, fmt.Errorf("pin: %s", err)
}
dagnodes = append(dagnodes, dagnode)
}
var out []key.Key
for _, dagnode := range dagnodes {
k, err := dagnode.Key()
if err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
err = n.Pinning.Pin(ctx, dagnode, recursive)
if err != nil {
return nil, fmt.Errorf("pin: %s", err)
}
out = append(out, k)
}
err := n.Pinning.Flush()
if err != nil {
return nil, err
}
return out, nil
}
func TestDagTruncate(t *testing.T) {
dserv, pins := getMockDagServ(t)
b, n := getNode(t, dserv, 50000, pins)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dagmod, err := NewDagModifier(ctx, n, dserv, pins, sizeSplitterGen(512))
if err != nil {
t.Fatal(err)
}
err = dagmod.Truncate(12345)
if err != nil {
t.Fatal(err)
}
_, err = dagmod.Seek(0, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(dagmod)
if err != nil {
t.Fatal(err)
}
if err = arrComp(out, b[:12345]); err != nil {
t.Fatal(err)
}
}
// InitializeKeyspace sets the ipns record for the given key to
// point to an empty directory.
func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error {
emptyDir := &mdag.Node{Data: ft.FolderPBData()}
nodek, err := n.DAG.Add(emptyDir)
if err != nil {
return err
}
ctx, cancel := context.WithCancel(n.Context())
defer cancel()
err = n.Pinning.Pin(ctx, emptyDir, false)
if err != nil {
return err
}
err = n.Pinning.Flush()
if err != nil {
return err
}
pub := nsys.NewRoutingPublisher(n.Routing)
if err := pub.Publish(ctx, key, path.FromKey(nodek)); err != nil {
return err
}
return nil
}
func Unpin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]key.Key, error) {
dagnodes := make([]*merkledag.Node, 0)
for _, fpath := range paths {
dagnode, err := core.Resolve(ctx, n, path.Path(fpath))
if err != nil {
return nil, err
}
dagnodes = append(dagnodes, dagnode)
}
var unpinned []key.Key
for _, dagnode := range dagnodes {
k, _ := dagnode.Key()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
err := n.Pinning.Unpin(ctx, k, recursive)
if err != nil {
return nil, err
}
unpinned = append(unpinned, k)
}
err := n.Pinning.Flush()
if err != nil {
return nil, err
}
return unpinned, nil
}
func (bs *Bitswap) connectToProviders(ctx context.Context, entries []wantlist.Entry) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
// Get providers for all entries in wantlist (could take a while)
wg := sync.WaitGroup{}
for _, e := range entries {
wg.Add(1)
go func(k key.Key) {
defer wg.Done()
child, cancel := context.WithTimeout(ctx, providerRequestTimeout)
defer cancel()
providers := bs.network.FindProvidersAsync(child, k, maxProvidersPerRequest)
for prov := range providers {
go func(p peer.ID) {
bs.network.ConnectTo(ctx, p)
}(prov)
}
}(e.Key)
}
wg.Wait() // make sure all our children do finish.
}
func addDefaultAssets(out io.Writer, repoRoot string) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
r, err := fsrepo.Open(repoRoot)
if err != nil { // NB: repo is owned by the node
return err
}
nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r})
if err != nil {
return err
}
defer nd.Close()
dkey, err := assets.SeedInitDocs(nd)
if err != nil {
return fmt.Errorf("init: seeding init docs failed: %s", err)
}
log.Debugf("init: seeded init docs %s", dkey)
if _, err = fmt.Fprintf(out, "to get started, enter:\n"); err != nil {
return err
}
_, err = fmt.Fprintf(out, "\n\tipfs cat /ipfs/%s/readme\n\n", dkey)
return err
}
func (dm *DagModifier) readPrep() error {
err := dm.Sync()
if err != nil {
return err
}
if dm.read == nil {
ctx, cancel := context.WithCancel(dm.ctx)
dr, err := uio.NewDagReader(ctx, dm.curNode, dm.dagserv)
if err != nil {
return err
}
i, err := dr.Seek(int64(dm.curWrOff), os.SEEK_SET)
if err != nil {
return err
}
if i != int64(dm.curWrOff) {
return ErrSeekFail
}
dm.readCancel = cancel
dm.read = dr
}
return nil
}
// WithDeadlineFraction returns a Context with a fraction of the
// original context's timeout. This is useful in sequential pipelines
// of work, where one might try options and fall back to others
// depending on the time available, or failure to respond. For example:
//
// // getPicture returns a picture from our encrypted database
// // we have a pipeline of multiple steps. we need to:
// // - get the data from a database
// // - decrypt it
// // - apply many transforms
// //
// // we **know** that each step takes increasingly more time.
// // The transforms are much more expensive than decryption, and
// // decryption is more expensive than the database lookup.
// // If our database takes too long (i.e. >0.2 of available time),
// // there's no use in continuing.
// func getPicture(ctx context.Context, key string) ([]byte, error) {
// // fractional timeout contexts to the rescue!
//
// // try the database with 0.2 of remaining time.
// ctx1, _ := ctxext.WithDeadlineFraction(ctx, 0.2)
// val, err := db.Get(ctx1, key)
// if err != nil {
// return nil, err
// }
//
// // try decryption with 0.3 of remaining time.
// ctx2, _ := ctxext.WithDeadlineFraction(ctx, 0.3)
// if val, err = decryptor.Decrypt(ctx2, val); err != nil {
// return nil, err
// }
//
// // try transforms with all remaining time. hopefully it's enough!
// return transformer.Transform(ctx, val)
// }
//
//
func WithDeadlineFraction(ctx context.Context, fraction float64) (
context.Context, context.CancelFunc) {
d, found := ctx.Deadline()
if !found { // no deadline
return context.WithCancel(ctx)
}
left := d.Sub(time.Now())
if left < 0 { // already passed...
return context.WithCancel(ctx)
}
left = time.Duration(float64(left) * fraction)
return context.WithTimeout(ctx, left)
}
func (i *cmdInvocation) SetupInterruptHandler(ctx context.Context) (io.Closer, context.Context) {
intrh := NewIntrHandler()
ctx, cancelFunc := context.WithCancel(ctx)
handlerFunc := func(count int, ih *IntrHandler) {
switch count {
case 1:
fmt.Println() // Prevent un-terminated ^C character in terminal
ih.wg.Add(1)
go func() {
defer ih.wg.Done()
cancelFunc()
}()
default:
fmt.Println("Received another interrupt before graceful shutdown, terminating...")
os.Exit(-1)
}
}
intrh.Handle(handlerFunc, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM)
return intrh, ctx
}
// WithProcessClosing returns a context.Context derived from ctx that
// is cancelled as p is Closing (after: <-p.Closing()). It is simply:
//
// func WithProcessClosing(ctx context.Context, p goprocess.Process) context.Context {
// ctx, cancel := context.WithCancel(ctx)
// go func() {
// <-p.Closing()
// cancel()
// }()
// return ctx
// }
//
func WithProcessClosing(ctx context.Context, p goprocess.Process) context.Context {
ctx, cancel := context.WithCancel(ctx)
go func() {
<-p.Closing()
cancel()
}()
return ctx
}
func addNode(n *core.IpfsNode, node *merkledag.Node) error {
if err := n.DAG.AddRecursive(node); err != nil { // add the file to the graph + local storage
return err
}
ctx, cancel := context.WithCancel(n.Context())
defer cancel()
err := n.Pinning.Pin(ctx, node, true) // ensure we keep it
return err
}
// TODO does dht ensure won't receive self as a provider? probably not.
func TestCanceledContext(t *testing.T) {
rs := NewServer()
k := key.Key("hello")
// avoid leaking goroutine, without using the context to signal
// (we want the goroutine to keep trying to publish on a
// cancelled context until we've tested it doesnt do anything.)
done := make(chan struct{})
defer func() { done <- struct{}{} }()
t.Log("async'ly announce infinite stream of providers for key")
i := 0
go func() { // infinite stream
for {
select {
case <-done:
t.Log("exiting async worker")
return
default:
}
pi, err := testutil.RandIdentity()
if err != nil {
t.Error(err)
}
err = rs.Client(pi).Provide(context.Background(), k)
if err != nil {
t.Error(err)
}
i++
}
}()
local := testutil.RandIdentityOrFatal(t)
client := rs.Client(local)
t.Log("warning: max is finite so this test is non-deterministic")
t.Log("context cancellation could simply take lower priority")
t.Log("and result in receiving the max number of results")
max := 1000
t.Log("cancel the context before consuming")
ctx, cancelFunc := context.WithCancel(context.Background())
cancelFunc()
providers := client.FindProvidersAsync(ctx, k, max)
numProvidersReturned := 0
for _ = range providers {
numProvidersReturned++
}
t.Log(numProvidersReturned)
if numProvidersReturned == max {
t.Fatal("Context cancel had no effect")
}
}
// WARNING: this uses RandTestBogusIdentity DO NOT USE for NON TESTS!
func NewTestSessionGenerator(
net tn.Network) SessionGenerator {
ctx, cancel := context.WithCancel(context.Background())
return SessionGenerator{
net: net,
seq: 0,
ctx: ctx, // TODO take ctx as param to Next, Instances
cancel: cancel,
}
}
func TestDagModifierBasic(t *testing.T) {
dserv, pin := getMockDagServ(t)
b, n := getNode(t, dserv, 50000, pin)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dagmod, err := NewDagModifier(ctx, n, dserv, pin, sizeSplitterGen(512))
if err != nil {
t.Fatal(err)
}
// Within zero block
beg := uint64(15)
length := uint64(60)
t.Log("Testing mod within zero block")
b = testModWrite(t, beg, length, b, dagmod)
// Within bounds of existing file
beg = 1000
length = 4000
t.Log("Testing mod within bounds of existing multiblock file.")
b = testModWrite(t, beg, length, b, dagmod)
// Extend bounds
beg = 49500
length = 4000
t.Log("Testing mod that extends file.")
b = testModWrite(t, beg, length, b, dagmod)
// "Append"
beg = uint64(len(b))
length = 3000
t.Log("Testing pure append")
b = testModWrite(t, beg, length, b, dagmod)
// Verify reported length
node, err := dagmod.GetNode()
if err != nil {
t.Fatal(err)
}
size, err := ft.DataSize(node.Data)
if err != nil {
t.Fatal(err)
}
expected := uint64(50000 + 3500 + 3000)
if size != expected {
t.Fatalf("Final reported size is incorrect [%d != %d]", size, expected)
}
}
func TestDoReturnsContextErr(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
ch := make(chan struct{})
err := ContextDo(ctx, func() error {
cancel()
ch <- struct{}{} // won't return
return nil
})
if err != ctx.Err() {
t.Fail()
}
}
func TestMultiWrite(t *testing.T) {
dserv, pins := getMockDagServ(t)
_, n := getNode(t, dserv, 0, pins)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dagmod, err := NewDagModifier(ctx, n, dserv, pins, sizeSplitterGen(512))
if err != nil {
t.Fatal(err)
}
data := make([]byte, 4000)
u.NewTimeSeededRand().Read(data)
for i := 0; i < len(data); i++ {
n, err := dagmod.WriteAt(data[i:i+1], int64(i))
if err != nil {
t.Fatal(err)
}
if n != 1 {
t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)")
}
size, err := dagmod.Size()
if err != nil {
t.Fatal(err)
}
if size != int64(i+1) {
t.Fatal("Size was reported incorrectly")
}
}
nd, err := dagmod.GetNode()
if err != nil {
t.Fatal(err)
}
read, err := uio.NewDagReader(context.Background(), nd, dserv)
if err != nil {
t.Fatal(err)
}
rbuf, err := ioutil.ReadAll(read)
if err != nil {
t.Fatal(err)
}
err = arrComp(rbuf, data)
if err != nil {
t.Fatal(err)
}
}
func NewDataFileReader(ctx context.Context, n *mdag.Node, pb *ftpb.Data, serv mdag.DAGService) *DagReader {
fctx, cancel := context.WithCancel(ctx)
promises := serv.GetDAG(fctx, n)
return &DagReader{
node: n,
serv: serv,
buf: NewRSNCFromBytes(pb.GetData()),
promises: promises,
ctx: fctx,
cancel: cancel,
pbdata: pb,
}
}
// Run runs the query at hand. pass in a list of peers to use first.
func (q *dhtQuery) Run(ctx context.Context, peers []peer.ID) (*dhtQueryResult, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
runner := newQueryRunner(q)
return runner.Run(ctx, peers)
}
func TestReaderCancel(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
piper, pipew := io.Pipe()
r := NewReader(ctx, piper)
buf := make([]byte, 10)
done := make(chan ioret)
go func() {
n, err := r.Read(buf)
done <- ioret{n, err}
}()
pipew.Write([]byte("abcdefghij"))
select {
case ret := <-done:
if ret.n != 10 {
t.Error("ret.n should be 10", ret.n)
}
if ret.err != nil {
t.Error("ret.err should be nil", ret.err)
}
if string(buf) != "abcdefghij" {
t.Error("read contents differ")
}
case <-time.After(20 * time.Millisecond):
t.Fatal("failed to read")
}
go func() {
n, err := r.Read(buf)
done <- ioret{n, err}
}()
cancel()
select {
case ret := <-done:
if ret.n != 0 {
t.Error("ret.n should be 0", ret.n)
}
if ret.err == nil {
t.Error("ret.err should be ctx error", ret.err)
}
case <-time.After(20 * time.Millisecond):
t.Fatal("failed to stop reading after cancel")
}
}
func runReadBench(b *testing.B, nd *dag.Node, ds dag.DAGService) {
for i := 0; i < b.N; i++ {
ctx, cancel := context.WithCancel(context.Background())
read, err := uio.NewDagReader(ctx, nd, ds)
if err != nil {
b.Fatal(err)
}
_, err = read.WriteTo(ioutil.Discard)
if err != nil && err != io.EOF {
b.Fatal(err)
}
cancel()
}
}
func basicGC(t *testing.T, bs blockstore.Blockstore, pins pin.ManualPinner) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel() // in case error occurs during operation
keychan, err := bs.AllKeysChan(ctx)
if err != nil {
t.Fatal(err)
}
for k := range keychan { // rely on AllKeysChan to close chan
if !pins.IsPinned(k) {
err := bs.DeleteBlock(k)
if err != nil {
t.Fatal(err)
}
}
}
}
// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) {
if n == nil {
return nil, fmt.Errorf("dagService is nil")
}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
b, err := n.Blocks.GetBlock(ctx, k)
if err != nil {
if err == bserv.ErrNotFound {
return nil, ErrNotFound
}
return nil, err
}
return Decoded(b.Data)
}