func WithDeadlineFraction(ctx context.Context, fraction float64) (context.Context, context.CancelFunc) {
d, found := ctx.Deadline()
if !found { // no deadline
return context.WithCancel(ctx)
}
left := d.Sub(time.Now())
if left < 0 { // already passed...
return context.WithCancel(ctx)
}
left = time.Duration(float64(left) * fraction)
return context.WithTimeout(ctx, left)
}
// InitializeKeyspace sets the ipns record for the given key to
// point to an empty directory.
func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error {
emptyDir := &mdag.Node{Data: ft.FolderPBData()}
nodek, err := n.DAG.Add(emptyDir)
if err != nil {
return err
}
ctx, cancel := context.WithCancel(n.Context())
defer cancel()
err = n.Pinning.Pin(ctx, emptyDir, false)
if err != nil {
return err
}
err = n.Pinning.Flush()
if err != nil {
return err
}
pub := nsys.NewRoutingPublisher(n.Routing, n.Repo.Datastore())
if err := pub.Publish(ctx, key, path.FromKey(nodek)); err != nil {
return err
}
return nil
}
func (i *cmdInvocation) SetupInterruptHandler(ctx context.Context) (io.Closer, context.Context) {
intrh := NewIntrHandler()
ctx, cancelFunc := context.WithCancel(ctx)
handlerFunc := func(count int, ih *IntrHandler) {
switch count {
case 1:
fmt.Println() // Prevent un-terminated ^C character in terminal
ih.wg.Add(1)
go func() {
defer ih.wg.Done()
cancelFunc()
}()
default:
fmt.Println("Received another interrupt before graceful shutdown, terminating...")
os.Exit(-1)
}
}
intrh.Handle(handlerFunc, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM)
return intrh, ctx
}
func (rp *Republisher) republishEntries(p goprocess.Process) error {
ctx, cancel := context.WithCancel(gpctx.OnClosingContext(p))
defer cancel()
for id, _ := range rp.entries {
log.Debugf("republishing ipns entry for %s", id)
priv := rp.ps.PrivKey(id)
// Look for it locally only
_, ipnskey := namesys.IpnsKeysForID(id)
p, seq, err := rp.getLastVal(ipnskey)
if err != nil {
if err == errNoEntry {
continue
}
return err
}
// update record with same sequence number
eol := time.Now().Add(rp.RecordLifetime)
err = namesys.PutRecordToRouting(ctx, priv, p, seq, eol, rp.r, id)
if err != nil {
return err
}
}
return nil
}
func TestBasic(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ds, rt := setupRoot(ctx, t)
rootdir := rt.GetValue().(*Directory)
// test making a basic dir
_, err := rootdir.Mkdir("a")
if err != nil {
t.Fatal(err)
}
path := "a/b/c/d/e/f/g"
d := mkdirP(t, rootdir, path)
fi := getRandFile(t, ds, 1000)
// test inserting that file
err = d.AddChild("afile", fi)
if err != nil {
t.Fatal(err)
}
err = assertFileAtPath(ds, rootdir, fi, "a/b/c/d/e/f/g/afile")
if err != nil {
t.Fatal(err)
}
}
func Unpin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]key.Key, error) {
var unpinned []key.Key
for _, p := range paths {
p, err := path.ParsePath(p)
if err != nil {
return nil, err
}
k, err := core.ResolveToKey(ctx, n, p)
if err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
err = n.Pinning.Unpin(ctx, k, recursive)
if err != nil {
return nil, err
}
unpinned = append(unpinned, k)
}
err := n.Pinning.Flush()
if err != nil {
return nil, err
}
return unpinned, nil
}
func (bs *Bitswap) connectToProviders(ctx context.Context, entries []wantlist.Entry) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
// Get providers for all entries in wantlist (could take a while)
wg := sync.WaitGroup{}
for _, e := range entries {
wg.Add(1)
go func(k key.Key) {
defer wg.Done()
child, cancel := context.WithTimeout(ctx, providerRequestTimeout)
defer cancel()
providers := bs.network.FindProvidersAsync(child, k, maxProvidersPerRequest)
for prov := range providers {
go func(p peer.ID) {
bs.network.ConnectTo(ctx, p)
}(prov)
}
}(e.Key)
}
wg.Wait() // make sure all our children do finish.
}
func (dm *DagModifier) readPrep() error {
err := dm.Sync()
if err != nil {
return err
}
if dm.read == nil {
ctx, cancel := context.WithCancel(dm.ctx)
dr, err := uio.NewDagReader(ctx, dm.curNode, dm.dagserv)
if err != nil {
return err
}
i, err := dr.Seek(int64(dm.curWrOff), os.SEEK_SET)
if err != nil {
return err
}
if i != int64(dm.curWrOff) {
return ErrSeekFail
}
dm.readCancel = cancel
dm.read = dr
}
return nil
}
func Unpin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]key.Key, error) {
dagnodes := make([]*merkledag.Node, 0)
for _, fpath := range paths {
dagnode, err := core.Resolve(ctx, n, path.Path(fpath))
if err != nil {
return nil, err
}
dagnodes = append(dagnodes, dagnode)
}
var unpinned []key.Key
for _, dagnode := range dagnodes {
k, _ := dagnode.Key()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
err := n.Pinning.Unpin(ctx, k, recursive)
if err != nil {
return nil, err
}
unpinned = append(unpinned, k)
}
err := n.Pinning.Flush()
if err != nil {
return nil, err
}
return unpinned, nil
}
func RunSupernodePutRecordGetRecord(conf testutil.LatencyConfig) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
servers, clients, err := InitializeSupernodeNetwork(ctx, 2, 2, conf)
if err != nil {
return err
}
for _, n := range append(servers, clients...) {
defer n.Close()
}
putter := clients[0]
getter := clients[1]
k := key.Key("key")
note := []byte("a note from putter")
if err := putter.Routing.PutValue(ctx, k, note); err != nil {
return fmt.Errorf("failed to put value: %s", err)
}
received, err := getter.Routing.GetValue(ctx, k)
if err != nil {
return fmt.Errorf("failed to get value: %s", err)
}
if 0 != bytes.Compare(note, received) {
return errors.New("record doesn't match")
}
return nil
}
func TestDoesNotDeadLockIfContextCancelledBeforePublish(t *testing.T) {
g := blocksutil.NewBlockGenerator()
ctx, cancel := context.WithCancel(context.Background())
n := New()
defer n.Shutdown()
t.Log("generate a large number of blocks. exceed default buffer")
bs := g.Blocks(1000)
ks := func() []key.Key {
var keys []key.Key
for _, b := range bs {
keys = append(keys, b.Key())
}
return keys
}()
_ = n.Subscribe(ctx, ks...) // ignore received channel
t.Log("cancel context before any blocks published")
cancel()
for _, b := range bs {
n.Publish(b)
}
t.Log("publishing the large number of blocks to the ignored channel must not deadlock")
}
// newKeyRoot creates a new KeyRoot for the given key, and starts up a republisher routine
// for it
func (fs *Filesystem) newKeyRoot(parent context.Context, k ci.PrivKey) (*KeyRoot, error) {
hash, err := k.GetPublic().Hash()
if err != nil {
return nil, err
}
name := "/ipns/" + key.Key(hash).String()
root := new(KeyRoot)
root.key = k
root.fs = fs
root.name = name
ctx, cancel := context.WithCancel(parent)
defer cancel()
pointsTo, err := fs.nsys.Resolve(ctx, name)
if err != nil {
err = namesys.InitializeKeyspace(ctx, fs.dserv, fs.nsys, fs.pins, k)
if err != nil {
return nil, err
}
pointsTo, err = fs.nsys.Resolve(ctx, name)
if err != nil {
return nil, err
}
}
mnode, err := fs.resolver.ResolvePath(ctx, pointsTo)
if err != nil {
log.Errorf("Failed to retrieve value '%s' for ipns entry: %s\n", pointsTo, err)
return nil, err
}
root.node = mnode
root.repub = NewRepublisher(root, time.Millisecond*300, time.Second*3)
go root.repub.Run(parent)
pbn, err := ft.FromBytes(mnode.Data)
if err != nil {
log.Error("IPNS pointer was not unixfs node")
return nil, err
}
switch pbn.GetType() {
case ft.TDirectory:
root.val = NewDirectory(ctx, pointsTo.String(), mnode, root, fs)
case ft.TFile, ft.TMetadata, ft.TRaw:
fi, err := NewFile(pointsTo.String(), mnode, root, fs)
if err != nil {
return nil, err
}
root.val = fi
default:
panic("unrecognized! (NYI)")
}
return root, nil
}
func (bs *Bitswap) rebroadcastWorker(parent context.Context) {
ctx, cancel := context.WithCancel(parent)
defer cancel()
broadcastSignal := time.NewTicker(rebroadcastDelay.Get())
defer broadcastSignal.Stop()
tick := time.NewTicker(10 * time.Second)
defer tick.Stop()
for {
log.Event(ctx, "Bitswap.Rebroadcast.idle")
select {
case <-tick.C:
n := bs.wm.wl.Len()
if n > 0 {
log.Debug(n, "keys in bitswap wantlist")
}
case <-broadcastSignal.C: // resend unfulfilled wantlist keys
log.Event(ctx, "Bitswap.Rebroadcast.active")
entries := bs.wm.wl.Entries()
if len(entries) > 0 {
bs.connectToProviders(ctx, entries)
}
case <-parent.Done():
return
}
}
}
func TestConsistentAccounting(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sender := newEngine(ctx, "Ernie")
receiver := newEngine(ctx, "Bert")
// Send messages from Ernie to Bert
for i := 0; i < 1000; i++ {
m := message.New(false)
content := []string{"this", "is", "message", "i"}
m.AddBlock(blocks.NewBlock([]byte(strings.Join(content, " "))))
sender.Engine.MessageSent(receiver.Peer, m)
receiver.Engine.MessageReceived(sender.Peer, m)
}
// Ensure sender records the change
if sender.Engine.numBytesSentTo(receiver.Peer) == 0 {
t.Fatal("Sent bytes were not recorded")
}
// Ensure sender and receiver have the same values
if sender.Engine.numBytesSentTo(receiver.Peer) != receiver.Engine.numBytesReceivedFrom(sender.Peer) {
t.Fatal("Inconsistent book-keeping. Strategies don't agree")
}
// Ensure sender didn't record receving anything. And that the receiver
// didn't record sending anything
if receiver.Engine.numBytesSentTo(sender.Peer) != 0 || sender.Engine.numBytesReceivedFrom(receiver.Peer) != 0 {
t.Fatal("Bert didn't send bytes to Ernie")
}
}
func DirectAddCat(data []byte, conf testutil.LatencyConfig) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
const numPeers = 2
// create network
mn, err := mocknet.FullMeshLinked(ctx, numPeers)
if err != nil {
return err
}
mn.SetLinkDefaults(mocknet.LinkOptions{
Latency: conf.NetworkLatency,
// TODO add to conf. This is tricky because we want 0 values to be functional.
Bandwidth: math.MaxInt32,
})
peers := mn.Peers()
if len(peers) < numPeers {
return errors.New("test initialization error")
}
adder, err := core.NewIPFSNode(ctx, core.ConfigOption(MocknetTestRepo(peers[0], mn.Host(peers[0]), conf, core.DHTOption)))
if err != nil {
return err
}
defer adder.Close()
catter, err := core.NewIPFSNode(ctx, core.ConfigOption(MocknetTestRepo(peers[1], mn.Host(peers[1]), conf, core.DHTOption)))
if err != nil {
return err
}
defer catter.Close()
bs1 := []peer.PeerInfo{adder.Peerstore.PeerInfo(adder.Identity)}
bs2 := []peer.PeerInfo{catter.Peerstore.PeerInfo(catter.Identity)}
if err := catter.Bootstrap(core.BootstrapConfigWithPeers(bs1)); err != nil {
return err
}
if err := adder.Bootstrap(core.BootstrapConfigWithPeers(bs2)); err != nil {
return err
}
added, err := coreunix.Add(adder, bytes.NewReader(data))
if err != nil {
return err
}
readerCatted, err := coreunix.Cat(catter, added)
if err != nil {
return err
}
// verify
bufout := new(bytes.Buffer)
io.Copy(bufout, readerCatted)
if 0 != bytes.Compare(bufout.Bytes(), data) {
return errors.New("catted data does not match added data")
}
return nil
}
func addDefaultAssets(out io.Writer, repoRoot string) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
r, err := fsrepo.Open(repoRoot)
if err != nil { // NB: repo is owned by the node
return err
}
nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r})
if err != nil {
return err
}
defer nd.Close()
dkey, err := assets.SeedInitDocs(nd)
if err != nil {
return fmt.Errorf("init: seeding init docs failed: %s", err)
}
log.Debugf("init: seeded init docs %s", dkey)
if _, err = fmt.Fprintf(out, "to get started, enter:\n"); err != nil {
return err
}
_, err = fmt.Fprintf(out, "\n\tipfs cat /ipfs/%s/readme\n\n", dkey)
return err
}
func TestReadPostCancel(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
piper, pipew := io.Pipe()
r := NewReader(ctx, piper)
buf := make([]byte, 10)
done := make(chan ioret)
go func() {
n, err := r.Read(buf)
done <- ioret{n, err}
}()
cancel()
select {
case ret := <-done:
if ret.n != 0 {
t.Error("ret.n should be 0", ret.n)
}
if ret.err == nil {
t.Error("ret.err should be ctx error", ret.err)
}
case <-time.After(20 * time.Millisecond):
t.Fatal("failed to stop reading after cancel")
}
pipew.Write([]byte("abcdefghij"))
if !bytes.Equal(buf, make([]byte, len(buf))) {
t.Fatal("buffer should have not been written to")
}
}
func Pin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]key.Key, error) {
dagnodes := make([]*merkledag.Node, 0)
for _, fpath := range paths {
dagnode, err := core.Resolve(ctx, n, path.Path(fpath))
if err != nil {
return nil, fmt.Errorf("pin: %s", err)
}
dagnodes = append(dagnodes, dagnode)
}
var out []key.Key
for _, dagnode := range dagnodes {
k, err := dagnode.Key()
if err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
err = n.Pinning.Pin(ctx, dagnode, recursive)
if err != nil {
return nil, fmt.Errorf("pin: %s", err)
}
out = append(out, k)
}
err := n.Pinning.Flush()
if err != nil {
return nil, err
}
return out, nil
}
func BenchmarkDagmodWrite(b *testing.B) {
b.StopTimer()
dserv, pins := getMockDagServ(b)
_, n := getNode(b, dserv, 0, pins)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
wrsize := 4096
dagmod, err := NewDagModifier(ctx, n, dserv, pins, sizeSplitterGen(512))
if err != nil {
b.Fatal(err)
}
buf := make([]byte, b.N*wrsize)
u.NewTimeSeededRand().Read(buf)
b.StartTimer()
b.SetBytes(int64(wrsize))
for i := 0; i < b.N; i++ {
n, err := dagmod.Write(buf[i*wrsize : (i+1)*wrsize])
if err != nil {
b.Fatal(err)
}
if n != wrsize {
b.Fatal("Wrote bad size")
}
}
}
func TestDagTruncate(t *testing.T) {
dserv, pins := getMockDagServ(t)
b, n := getNode(t, dserv, 50000, pins)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dagmod, err := NewDagModifier(ctx, n, dserv, pins, sizeSplitterGen(512))
if err != nil {
t.Fatal(err)
}
err = dagmod.Truncate(12345)
if err != nil {
t.Fatal(err)
}
_, err = dagmod.Seek(0, os.SEEK_SET)
if err != nil {
t.Fatal(err)
}
out, err := ioutil.ReadAll(dagmod)
if err != nil {
t.Fatal(err)
}
if err = arrComp(out, b[:12345]); err != nil {
t.Fatal(err)
}
}
func TestWritePostCancel(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
piper, pipew := io.Pipe()
w := NewWriter(ctx, pipew)
buf := []byte("abcdefghij")
buf2 := make([]byte, 10)
done := make(chan ioret)
go func() {
n, err := w.Write(buf)
done <- ioret{n, err}
}()
piper.Read(buf2)
select {
case ret := <-done:
if ret.n != 10 {
t.Error("ret.n should be 10", ret.n)
}
if ret.err != nil {
t.Error("ret.err should be nil", ret.err)
}
if string(buf2) != "abcdefghij" {
t.Error("write contents differ")
}
case <-time.After(20 * time.Millisecond):
t.Fatal("failed to write")
}
go func() {
n, err := w.Write(buf)
done <- ioret{n, err}
}()
cancel()
select {
case ret := <-done:
if ret.n != 0 {
t.Error("ret.n should be 0", ret.n)
}
if ret.err == nil {
t.Error("ret.err should be ctx error", ret.err)
}
case <-time.After(20 * time.Millisecond):
t.Fatal("failed to stop writing after cancel")
}
copy(buf, []byte("aaaaaaaaaa"))
piper.Read(buf2)
if string(buf2) == "aaaaaaaaaa" {
t.Error("buffer was read from after ctx cancel")
} else if string(buf2) != "abcdefghij" {
t.Error("write contents differ from expected")
}
}
// Context returns a context that cancels when the waitable is closing.
func Context(w Waitable) context.Context {
ctx, cancel := context.WithCancel(context.Background())
go func() {
<-w.Closing()
cancel()
}()
return ctx
}
// WithProcessClosed returns a context.Context that is cancelled
// after Process p is Closed. It is the equivalent of:
//
// func WithProcessClosed(ctx context.Context, p goprocess.Process) context.Context {
// ctx, cancel := context.WithCancel(ctx)
// go func() {
// <-p.Closed()
// cancel()
// }()
// return ctx
// }
//
func WithProcessClosed(ctx context.Context, p goprocess.Process) context.Context {
ctx, cancel := context.WithCancel(ctx)
go func() {
<-p.Closed()
cancel()
}()
return ctx
}
func addNode(n *core.IpfsNode, node *merkledag.Node) error {
if err := n.DAG.AddRecursive(node); err != nil { // add the file to the graph + local storage
return err
}
ctx, cancel := context.WithCancel(n.Context())
defer cancel()
err := n.Pinning.Pin(ctx, node, true) // ensure we keep it
return err
}
// TODO does dht ensure won't receive self as a provider? probably not.
func TestCanceledContext(t *testing.T) {
rs := NewServer()
k := key.Key("hello")
// avoid leaking goroutine, without using the context to signal
// (we want the goroutine to keep trying to publish on a
// cancelled context until we've tested it doesnt do anything.)
done := make(chan struct{})
defer func() { done <- struct{}{} }()
t.Log("async'ly announce infinite stream of providers for key")
i := 0
go func() { // infinite stream
for {
select {
case <-done:
t.Log("exiting async worker")
return
default:
}
pi, err := testutil.RandIdentity()
if err != nil {
t.Error(err)
}
err = rs.Client(pi).Provide(context.Background(), k)
if err != nil {
t.Error(err)
}
i++
}
}()
local := testutil.RandIdentityOrFatal(t)
client := rs.Client(local)
t.Log("warning: max is finite so this test is non-deterministic")
t.Log("context cancellation could simply take lower priority")
t.Log("and result in receiving the max number of results")
max := 1000
t.Log("cancel the context before consuming")
ctx, cancelFunc := context.WithCancel(context.Background())
cancelFunc()
providers := client.FindProvidersAsync(ctx, k, max)
numProvidersReturned := 0
for _ = range providers {
numProvidersReturned++
}
t.Log(numProvidersReturned)
if numProvidersReturned == max {
t.Fatal("Context cancel had no effect")
}
}
// WARNING: this uses RandTestBogusIdentity DO NOT USE for NON TESTS!
func NewTestSessionGenerator(
net tn.Network) SessionGenerator {
ctx, cancel := context.WithCancel(context.TODO())
return SessionGenerator{
net: net,
seq: 0,
ctx: ctx, // TODO take ctx as param to Next, Instances
cancel: cancel,
}
}
func (i internalHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
log.Debug("Incoming API request: ", r.URL)
if !allowOrigin(r, i.cfg) || !allowReferer(r, i.cfg) {
w.WriteHeader(http.StatusForbidden)
w.Write([]byte("403 - Forbidden"))
log.Warningf("API blocked request to %s. (possible CSRF)", r.URL)
return
}
req, err := Parse(r, i.root)
if err != nil {
if err == ErrNotFound {
w.WriteHeader(http.StatusNotFound)
} else {
w.WriteHeader(http.StatusBadRequest)
}
w.Write([]byte(err.Error()))
return
}
// get the node's context to pass into the commands.
node, err := i.ctx.GetNode()
if err != nil {
s := fmt.Sprintf("cmds/http: couldn't GetNode(): %s", err)
http.Error(w, s, http.StatusInternalServerError)
return
}
//ps: take note of the name clash - commands.Context != context.Context
req.SetInvocContext(i.ctx)
ctx, cancel := context.WithCancel(node.Context())
defer cancel()
err = req.SetRootContext(ctx)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// call the command
res := i.root.Call(req)
// set user's headers first.
for k, v := range i.cfg.Headers {
if !skipAPIHeader(k) {
w.Header()[k] = v
}
}
// now handle responding to the client properly
sendResponse(w, r, res, req)
}
func verifyResolutionFails(nodes []*core.IpfsNode, key string) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
for _, n := range nodes {
_, err := n.Namesys.Resolve(ctx, key)
if err == nil {
return errors.New("expected resolution to fail")
}
}
return nil
}
func TestDagModifierBasic(t *testing.T) {
dserv, pin := getMockDagServ(t)
b, n := getNode(t, dserv, 50000, pin)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dagmod, err := NewDagModifier(ctx, n, dserv, pin, sizeSplitterGen(512))
if err != nil {
t.Fatal(err)
}
// Within zero block
beg := uint64(15)
length := uint64(60)
t.Log("Testing mod within zero block")
b = testModWrite(t, beg, length, b, dagmod)
// Within bounds of existing file
beg = 1000
length = 4000
t.Log("Testing mod within bounds of existing multiblock file.")
b = testModWrite(t, beg, length, b, dagmod)
// Extend bounds
beg = 49500
length = 4000
t.Log("Testing mod that extends file.")
b = testModWrite(t, beg, length, b, dagmod)
// "Append"
beg = uint64(len(b))
length = 3000
t.Log("Testing pure append")
b = testModWrite(t, beg, length, b, dagmod)
// Verify reported length
node, err := dagmod.GetNode()
if err != nil {
t.Fatal(err)
}
size, err := ft.DataSize(node.Data)
if err != nil {
t.Fatal(err)
}
expected := uint64(50000 + 3500 + 3000)
if size != expected {
t.Fatalf("Final reported size is incorrect [%d != %d]", size, expected)
}
}
func TestDoReturnsContextErr(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
ch := make(chan struct{})
err := ContextDo(ctx, func() error {
cancel()
ch <- struct{}{} // won't return
return nil
})
if err != ctx.Err() {
t.Fail()
}
}