func TestReturnsErrorWhenSizeNegative(t *testing.T) { bs := NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore())) _, err := bloomCached(bs, context.TODO(), 100, 1, -1) if err == nil { t.Fail() } _, err = bloomCached(bs, context.TODO(), -1, 1, 100) if err == nil { t.Fail() } }
func connect(args args) error { p, ps, err := setupPeer(args) if err != nil { return err } var conn net.Conn if args.listen { conn, err = Listen(args.localAddr) } else { conn, err = Dial(args.localAddr, args.remoteAddr) } if err != nil { return err } // log everything that goes through conn rwc := &logRW{n: "conn", rw: conn} // OK, let's setup the channel. sk := ps.PrivKey(p) sg := secio.SessionGenerator{LocalID: p, PrivateKey: sk} sess, err := sg.NewSession(context.TODO(), rwc) if err != nil { return err } out("remote peer id: %s", sess.RemotePeer()) netcat(sess.ReadWriter().(io.ReadWriteCloser)) return nil }
// pipeStream relays over a stream to a remote peer. It's like `cat` func (rs *RelayService) pipeStream(src, dst peer.ID, s inet.Stream) error { // TODO: find a good way to pass contexts into here nsctx, cancel := context.WithTimeout(context.TODO(), time.Second*30) defer cancel() s2, err := rs.openStreamToPeer(nsctx, dst) if err != nil { return fmt.Errorf("failed to open stream to peer: %s -- %s", dst, err) } cancel() // cancel here because this function might last a while if err := WriteHeader(s2, src, dst); err != nil { return err } // connect the series of tubes. done := make(chan retio, 2) go func() { n, err := io.Copy(s2, s) done <- retio{n, err} }() go func() { n, err := io.Copy(s, s2) done <- retio{n, err} }() r1 := <-done r2 := <-done log.Infof("%s relayed %d/%d bytes between %s and %s", rs.host.ID(), r1.n, r2.n, src, dst) if r1.err != nil { return r1.err } return r2.err }
func TestFetchGraph(t *testing.T) { var dservs []DAGService bsis := bstest.Mocks(2) for _, bsi := range bsis { dservs = append(dservs, NewDAGService(bsi)) } read := io.LimitReader(u.NewTimeSeededRand(), 1024*32) root, err := imp.BuildDagFromReader(dservs[0], chunk.NewSizeSplitter(read, 512)) if err != nil { t.Fatal(err) } err = FetchGraph(context.TODO(), root, dservs[1]) if err != nil { t.Fatal(err) } // create an offline dagstore and ensure all blocks were fetched bs := bserv.New(bsis[1].Blockstore, offline.Exchange(bsis[1].Blockstore)) offline_ds := NewDAGService(bs) ks := key.NewKeySet() err = EnumerateChildren(context.Background(), offline_ds, root, ks, false) if err != nil { t.Fatal(err) } }
func (d *tcpDialer) reuseDial(raddr ma.Multiaddr) (manet.Conn, error) { logdial := lgbl.Dial("conn", "", "", d.laddr, raddr) rpev := log.EventBegin(context.TODO(), "tptDialReusePort", logdial) network, netraddr, err := manet.DialArgs(raddr) if err != nil { return nil, err } con, err := d.rd.Dial(network, netraddr) if err == nil { logdial["reuseport"] = "success" rpev.Done() return manet.WrapNetConn(con) } if !ReuseErrShouldRetry(err) { logdial["reuseport"] = "failure" logdial["error"] = err rpev.Done() return nil, err } logdial["reuseport"] = "retry" logdial["error"] = err rpev.Done() return d.madialer.Dial(raddr) }
func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) { fi.nodelk.Lock() node := fi.node fi.nodelk.Unlock() switch flags { case OpenReadOnly: fi.desclock.RLock() case OpenWriteOnly, OpenReadWrite: fi.desclock.Lock() default: // TODO: support other modes return nil, fmt.Errorf("mode not supported") } dmod, err := mod.NewDagModifier(context.TODO(), node, fi.dserv, chunk.DefaultSplitter) if err != nil { return nil, err } return &fileDescriptor{ inode: fi, perms: flags, sync: sync, mod: dmod, }, nil }
func (pn *peernet) Adapter(p testutil.Identity) bsnet.BitSwapNetwork { client, err := pn.Mocknet.AddPeer(p.PrivateKey(), p.Address()) if err != nil { panic(err.Error()) } routing := pn.routingserver.ClientWithDatastore(context.TODO(), p, ds.NewMapDatastore()) return bsnet.NewFromIpfsHost(client, routing) }
// Recursive call for verifying the structure of a trickledag func verifyTDagRec(nd *dag.Node, depth, direct, layerRepeat int, ds dag.DAGService) error { if depth == 0 { // zero depth dag is raw data block if len(nd.Links) > 0 { return errors.New("expected direct block") } pbn, err := ft.FromBytes(nd.Data) if err != nil { return err } if pbn.GetType() != ft.TRaw { return errors.New("Expected raw block") } return nil } // Verify this is a branch node pbn, err := ft.FromBytes(nd.Data) if err != nil { return err } if pbn.GetType() != ft.TFile { return errors.New("expected file as branch node") } if len(pbn.Data) > 0 { return errors.New("branch node should not have data") } for i := 0; i < len(nd.Links); i++ { child, err := nd.Links[i].GetNode(context.TODO(), ds) if err != nil { return err } if i < direct { // Direct blocks err := verifyTDagRec(child, 0, direct, layerRepeat, ds) if err != nil { return err } } else { // Recursive trickle dags rdepth := ((i - direct) / layerRepeat) + 1 if rdepth >= depth && depth > 0 { return errors.New("Child dag was too deep!") } err := verifyTDagRec(child, rdepth, direct, layerRepeat, ds) if err != nil { return err } } } return nil }
func (ids *IDService) IdentifyConn(c inet.Conn) { ids.currmu.Lock() if wait, found := ids.currid[c]; found { ids.currmu.Unlock() log.Debugf("IdentifyConn called twice on: %s", c) <-wait // already identifying it. wait for it. return } ch := make(chan struct{}) ids.currid[c] = ch ids.currmu.Unlock() defer close(ch) s, err := c.NewStream() if err != nil { log.Debugf("error opening initial stream for %s: %s", ID, err) log.Event(context.TODO(), "IdentifyOpenFailed", c.RemotePeer()) c.Close() return } bwc := ids.Host.GetBandwidthReporter() s = mstream.WrapStream(s, ID, bwc) // ok give the response to our handler. if err := msmux.SelectProtoOrFail(ID, s); err != nil { log.Debugf("error writing stream header for %s", ID) log.Event(context.TODO(), "IdentifyOpenFailed", c.RemotePeer()) s.Close() return } ids.ResponseHandler(s) ids.currmu.Lock() _, found := ids.currid[c] delete(ids.currid, c) ids.currmu.Unlock() if !found { log.Debugf("IdentifyConn failed to find channel (programmer error) for %s", c) return } }
func catNode(ds dag.DAGService, nd *dag.Node) ([]byte, error) { r, err := uio.NewDagReader(context.TODO(), nd, ds) if err != nil { return nil, err } defer r.Close() return ioutil.ReadAll(r) }
func logProtocolMismatchDisconnect(c inet.Conn, protocol, agent string) { lm := make(lgbl.DeferredMap) lm["remotePeer"] = func() interface{} { return c.RemotePeer().Pretty() } lm["remoteAddr"] = func() interface{} { return c.RemoteMultiaddr().String() } lm["protocolVersion"] = protocol lm["agentVersion"] = agent log.Event(context.TODO(), "IdentifyProtocolMismatch", lm) log.Debug("IdentifyProtocolMismatch %s %s %s (disconnected)", c.RemotePeer(), protocol, agent) }
func (w *writecache) Put(b blocks.Block) error { k := b.Key() if _, ok := w.cache.Get(k); ok { return nil } defer log.EventBegin(context.TODO(), "writecache.BlockAdded", &k).Done() w.cache.Add(b.Key(), struct{}{}) return w.blockstore.Put(b) }
func (n *network) deliver( r bsnet.Receiver, from peer.ID, message bsmsg.BitSwapMessage) error { if message == nil || from == "" { return errors.New("Invalid input") } n.delay.Wait() r.ReceiveMessage(context.TODO(), from, message) return nil }
func (w *writecache) PutMany(bs []blocks.Block) error { var good []blocks.Block for _, b := range bs { if _, ok := w.cache.Get(b.Key()); !ok { good = append(good, b) k := b.Key() defer log.EventBegin(context.TODO(), "writecache.BlockAdded", &k).Done() } } return w.blockstore.PutMany(good) }
func testBloomCached(bs GCBlockstore, ctx context.Context) (*bloomcache, error) { if ctx == nil { ctx = context.TODO() } opts := DefaultCacheOpts() bbs, err := CachedBlockstore(bs, ctx, opts) if err == nil { return bbs.(*bloomcache), nil } else { return nil, err } }
// LoadPinner loads a pinner and its keysets from the given datastore func LoadPinner(d ds.Datastore, dserv mdag.DAGService) (Pinner, error) { p := new(pinner) rootKeyI, err := d.Get(pinDatastoreKey) if err != nil { return nil, fmt.Errorf("cannot load pin state: %v", err) } rootKeyBytes, ok := rootKeyI.([]byte) if !ok { return nil, fmt.Errorf("cannot load pin state: %s was not bytes", pinDatastoreKey) } rootKey := key.Key(rootKeyBytes) ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5) defer cancel() root, err := dserv.Get(ctx, rootKey) if err != nil { return nil, fmt.Errorf("cannot find pinning root object: %v", err) } internalPin := map[key.Key]struct{}{ rootKey: struct{}{}, } recordInternal := func(k key.Key) { internalPin[k] = struct{}{} } { // load recursive set recurseKeys, err := loadSet(ctx, dserv, root, linkRecursive, recordInternal) if err != nil { return nil, fmt.Errorf("cannot load recursive pins: %v", err) } p.recursePin = set.SimpleSetFromKeys(recurseKeys) } { // load direct set directKeys, err := loadSet(ctx, dserv, root, linkDirect, recordInternal) if err != nil { return nil, fmt.Errorf("cannot load direct pins: %v", err) } p.directPin = set.SimpleSetFromKeys(directKeys) } p.internalPin = internalPin // assign services p.dserv = dserv p.dstore = d return p, nil }
// Flush encodes and writes pinner keysets to the datastore func (p *pinner) Flush() error { p.lock.Lock() defer p.lock.Unlock() ctx := context.TODO() internalPin := make(map[key.Key]struct{}) recordInternal := func(k key.Key) { internalPin[k] = struct{}{} } root := &mdag.Node{} { n, err := storeSet(ctx, p.dserv, p.directPin.GetKeys(), recordInternal) if err != nil { return err } if err := root.AddNodeLink(linkDirect, n); err != nil { return err } } { n, err := storeSet(ctx, p.dserv, p.recursePin.GetKeys(), recordInternal) if err != nil { return err } if err := root.AddNodeLink(linkRecursive, n); err != nil { return err } } // add the empty node, its referenced by the pin sets but never created _, err := p.dserv.Add(new(mdag.Node)) if err != nil { return err } k, err := p.dserv.Add(root) if err != nil { return err } internalPin[k] = struct{}{} if err := p.dstore.Put(pinDatastoreKey, []byte(k)); err != nil { return fmt.Errorf("cannot store pin state: %v", err) } p.internalPin = internalPin return nil }
func addPortMapping(nmgr *natManager, intaddr ma.Multiaddr) { nat := nmgr.NAT() if nat == nil { panic("natManager addPortMapping called without a nat.") } // first, check if the port mapping already exists. for _, mapping := range nat.Mappings() { if mapping.InternalAddr().Equal(intaddr) { return // it exists! return. } } ctx := context.TODO() lm := make(lgbl.DeferredMap) lm["internalAddr"] = func() interface{} { return intaddr.String() } defer log.EventBegin(ctx, "natMgrAddPortMappingWait", lm).Done() select { case <-nmgr.proc.Closing(): lm["outcome"] = "cancelled" return // no use. case <-nmgr.ready: // wait until it's ready. } // actually start the port map (sub-event because waiting may take a while) defer log.EventBegin(ctx, "natMgrAddPortMapping", lm).Done() // get the nat m, err := nat.NewMapping(intaddr) if err != nil { lm["outcome"] = "failure" lm["error"] = err return } extaddr, err := m.ExternalAddr() if err != nil { lm["outcome"] = "failure" lm["error"] = err return } lm["outcome"] = "success" lm["externalAddr"] = func() interface{} { return extaddr.String() } log.Infof("established nat port mapping: %s <--> %s", intaddr, extaddr) }
func (lb *Loopback) HandleStream(s inet.Stream) { defer s.Close() pbr := ggio.NewDelimitedReader(s, inet.MessageSizeMax) var incoming dhtpb.Message if err := pbr.ReadMsg(&incoming); err != nil { log.Debug(err) return } ctx := context.TODO() outgoing := lb.Handler.HandleRequest(ctx, s.Conn().RemotePeer(), &incoming) pbw := ggio.NewDelimitedWriter(s) if err := pbw.WriteMsg(outgoing); err != nil { return // TODO logerr } }
func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) { fi.nodelk.Lock() node := fi.node fi.nodelk.Unlock() fsn, err := ft.FSNodeFromBytes(node.Data()) if err != nil { return nil, err } switch fsn.Type { default: return nil, fmt.Errorf("unsupported fsnode type for 'file'") case ft.TSymlink: return nil, fmt.Errorf("symlinks not yet supported") case ft.TFile, ft.TRaw: // OK case } switch flags { case OpenReadOnly: fi.desclock.RLock() case OpenWriteOnly, OpenReadWrite: fi.desclock.Lock() default: // TODO: support other modes return nil, fmt.Errorf("mode not supported") } dmod, err := mod.NewDagModifier(context.TODO(), node, fi.dserv, chunk.DefaultSplitter) if err != nil { return nil, err } return &fileDescriptor{ inode: fi, perms: flags, sync: sync, mod: dmod, }, nil }
// getNode returns the node for link. If it return an error, // stop processing. if it returns a nil node, just skip it. // // the error handling is a little complicated. func (t *traversal) getNode(link *mdag.Link) (*mdag.Node, error) { getNode := func(l *mdag.Link) (*mdag.Node, error) { next, err := l.GetNode(context.TODO(), t.opts.DAG) if err != nil { return nil, err } skip, err := t.shouldSkip(next) if skip { next = nil } return next, err } next, err := getNode(link) if err != nil && t.opts.ErrFunc != nil { // attempt recovery. err = t.opts.ErrFunc(err) next = nil // skip regardless } return next, err }
// ResolveUnspecifiedAddresses expands unspecified ip addresses (/ip4/0.0.0.0, /ip6/::) to // use the known local interfaces. func ResolveUnspecifiedAddresses(unspecAddrs, ifaceAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) { // todo optimize: only fetch these if we have a "any" addr. if len(ifaceAddrs) < 1 { var err error ifaceAddrs, err = InterfaceAddresses() if err != nil { return nil, err } // log.Debug("InterfaceAddresses:", ifaceAddrs) } var outputAddrs []ma.Multiaddr for _, a := range unspecAddrs { // unspecified? resolved, err := ResolveUnspecifiedAddress(a, ifaceAddrs) if err != nil { continue // optimistic. if we cant resolve anything, we'll know at the bottom. } // log.Debug("resolved:", a, resolved) outputAddrs = append(outputAddrs, resolved...) } if len(outputAddrs) < 1 { return nil, fmt.Errorf("failed to specify addrs: %s", unspecAddrs) } log.Event(context.TODO(), "interfaceListenAddresses", func() logging.Loggable { var addrs []string for _, addr := range outputAddrs { addrs = append(addrs, addr.String()) } return logging.Metadata{"addresses": addrs} }()) log.Debug("ResolveUnspecifiedAddresses:", unspecAddrs, ifaceAddrs, outputAddrs) return outputAddrs, nil }
// Context returns the IpfsNode context func (n *IpfsNode) Context() context.Context { if n.ctx == nil { n.ctx = context.TODO() } return n.ctx }
func (pm *WantManager) CancelWants(ks []key.Key) { pm.addEntries(context.TODO(), ks, true) }
func TestRepublisher(t *testing.T) { if ci.IsRunning() { t.Skip("dont run timing tests in CI") } ctx := context.TODO() pub := make(chan struct{}) pf := func(ctx context.Context, k key.Key) error { pub <- struct{}{} return nil } tshort := time.Millisecond * 50 tlong := time.Second / 2 rp := NewRepublisher(ctx, pf, tshort, tlong) go rp.Run() rp.Update("test") // should hit short timeout select { case <-time.After(tshort * 2): t.Fatal("publish didnt happen in time") case <-pub: } cctx, cancel := context.WithCancel(context.Background()) go func() { for { rp.Update("a") time.Sleep(time.Millisecond * 10) select { case <-cctx.Done(): return default: } } }() select { case <-pub: t.Fatal("shouldnt have received publish yet!") case <-time.After((tlong * 9) / 10): } select { case <-pub: case <-time.After(tlong / 2): t.Fatal("waited too long for pub!") } cancel() go func() { err := rp.Close() if err != nil { t.Fatal(err) } }() // final pub from closing <-pub }
func (c *client) Send(req cmds.Request) (cmds.Response, error) { if req.Context() == nil { log.Warningf("no context set in request") if err := req.SetRootContext(context.TODO()); err != nil { return nil, err } } // save user-provided encoding previousUserProvidedEncoding, found, err := req.Option(cmds.EncShort).String() if err != nil { return nil, err } // override with json to send to server req.SetOption(cmds.EncShort, cmds.JSON) // stream channel output req.SetOption(cmds.ChanOpt, "true") query, err := getQuery(req) if err != nil { return nil, err } var fileReader *MultiFileReader var reader io.Reader if req.Files() != nil { fileReader = NewMultiFileReader(req.Files(), true) reader = fileReader } path := strings.Join(req.Path(), "/") url := fmt.Sprintf(ApiUrlFormat, c.serverAddress, ApiPath, path, query) httpReq, err := http.NewRequest("POST", url, reader) if err != nil { return nil, err } // TODO extract string consts? if fileReader != nil { httpReq.Header.Set(contentTypeHeader, "multipart/form-data; boundary="+fileReader.Boundary()) } else { httpReq.Header.Set(contentTypeHeader, applicationOctetStream) } httpReq.Header.Set(uaHeader, config.ApiVersion) httpReq.Cancel = req.Context().Done() httpReq.Close = true httpRes, err := c.httpClient.Do(httpReq) if err != nil { return nil, err } // using the overridden JSON encoding in request res, err := getResponse(httpRes, req) if err != nil { return nil, err } if found && len(previousUserProvidedEncoding) > 0 { // reset to user provided encoding after sending request // NB: if user has provided an encoding but it is the empty string, // still leave it as JSON. req.SetOption(cmds.EncShort, previousUserProvidedEncoding) } return res, nil }
func (rs *mocknetserver) Client(p testutil.Identity) Client { return rs.ClientWithDatastore(context.TODO(), p, ds.NewMapDatastore()) }
func (w *writecache) DeleteBlock(k key.Key) error { defer log.EventBegin(context.TODO(), "writecache.BlockRemoved", &k).Done() w.cache.Remove(k) return w.blockstore.DeleteBlock(k) }