func TestReturnsErrorWhenSizeNegative(t *testing.T) {
bs := NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
_, err := bloomCached(bs, context.TODO(), 100, 1, -1)
if err == nil {
t.Fail()
}
_, err = bloomCached(bs, context.TODO(), -1, 1, 100)
if err == nil {
t.Fail()
}
}
func connect(args args) error {
p, ps, err := setupPeer(args)
if err != nil {
return err
}
var conn net.Conn
if args.listen {
conn, err = Listen(args.localAddr)
} else {
conn, err = Dial(args.localAddr, args.remoteAddr)
}
if err != nil {
return err
}
// log everything that goes through conn
rwc := &logRW{n: "conn", rw: conn}
// OK, let's setup the channel.
sk := ps.PrivKey(p)
sg := secio.SessionGenerator{LocalID: p, PrivateKey: sk}
sess, err := sg.NewSession(context.TODO(), rwc)
if err != nil {
return err
}
out("remote peer id: %s", sess.RemotePeer())
netcat(sess.ReadWriter().(io.ReadWriteCloser))
return nil
}
// pipeStream relays over a stream to a remote peer. It's like `cat`
func (rs *RelayService) pipeStream(src, dst peer.ID, s inet.Stream) error {
// TODO: find a good way to pass contexts into here
nsctx, cancel := context.WithTimeout(context.TODO(), time.Second*30)
defer cancel()
s2, err := rs.openStreamToPeer(nsctx, dst)
if err != nil {
return fmt.Errorf("failed to open stream to peer: %s -- %s", dst, err)
}
cancel() // cancel here because this function might last a while
if err := WriteHeader(s2, src, dst); err != nil {
return err
}
// connect the series of tubes.
done := make(chan retio, 2)
go func() {
n, err := io.Copy(s2, s)
done <- retio{n, err}
}()
go func() {
n, err := io.Copy(s, s2)
done <- retio{n, err}
}()
r1 := <-done
r2 := <-done
log.Infof("%s relayed %d/%d bytes between %s and %s", rs.host.ID(), r1.n, r2.n, src, dst)
if r1.err != nil {
return r1.err
}
return r2.err
}
func TestFetchGraph(t *testing.T) {
var dservs []DAGService
bsis := bstest.Mocks(2)
for _, bsi := range bsis {
dservs = append(dservs, NewDAGService(bsi))
}
read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
root, err := imp.BuildDagFromReader(dservs[0], chunk.NewSizeSplitter(read, 512))
if err != nil {
t.Fatal(err)
}
err = FetchGraph(context.TODO(), root, dservs[1])
if err != nil {
t.Fatal(err)
}
// create an offline dagstore and ensure all blocks were fetched
bs := bserv.New(bsis[1].Blockstore, offline.Exchange(bsis[1].Blockstore))
offline_ds := NewDAGService(bs)
ks := key.NewKeySet()
err = EnumerateChildren(context.Background(), offline_ds, root, ks, false)
if err != nil {
t.Fatal(err)
}
}
func (d *tcpDialer) reuseDial(raddr ma.Multiaddr) (manet.Conn, error) {
logdial := lgbl.Dial("conn", "", "", d.laddr, raddr)
rpev := log.EventBegin(context.TODO(), "tptDialReusePort", logdial)
network, netraddr, err := manet.DialArgs(raddr)
if err != nil {
return nil, err
}
con, err := d.rd.Dial(network, netraddr)
if err == nil {
logdial["reuseport"] = "success"
rpev.Done()
return manet.WrapNetConn(con)
}
if !ReuseErrShouldRetry(err) {
logdial["reuseport"] = "failure"
logdial["error"] = err
rpev.Done()
return nil, err
}
logdial["reuseport"] = "retry"
logdial["error"] = err
rpev.Done()
return d.madialer.Dial(raddr)
}
func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) {
fi.nodelk.Lock()
node := fi.node
fi.nodelk.Unlock()
switch flags {
case OpenReadOnly:
fi.desclock.RLock()
case OpenWriteOnly, OpenReadWrite:
fi.desclock.Lock()
default:
// TODO: support other modes
return nil, fmt.Errorf("mode not supported")
}
dmod, err := mod.NewDagModifier(context.TODO(), node, fi.dserv, chunk.DefaultSplitter)
if err != nil {
return nil, err
}
return &fileDescriptor{
inode: fi,
perms: flags,
sync: sync,
mod: dmod,
}, nil
}
func (pn *peernet) Adapter(p testutil.Identity) bsnet.BitSwapNetwork {
client, err := pn.Mocknet.AddPeer(p.PrivateKey(), p.Address())
if err != nil {
panic(err.Error())
}
routing := pn.routingserver.ClientWithDatastore(context.TODO(), p, ds.NewMapDatastore())
return bsnet.NewFromIpfsHost(client, routing)
}
// Recursive call for verifying the structure of a trickledag
func verifyTDagRec(nd *dag.Node, depth, direct, layerRepeat int, ds dag.DAGService) error {
if depth == 0 {
// zero depth dag is raw data block
if len(nd.Links) > 0 {
return errors.New("expected direct block")
}
pbn, err := ft.FromBytes(nd.Data)
if err != nil {
return err
}
if pbn.GetType() != ft.TRaw {
return errors.New("Expected raw block")
}
return nil
}
// Verify this is a branch node
pbn, err := ft.FromBytes(nd.Data)
if err != nil {
return err
}
if pbn.GetType() != ft.TFile {
return errors.New("expected file as branch node")
}
if len(pbn.Data) > 0 {
return errors.New("branch node should not have data")
}
for i := 0; i < len(nd.Links); i++ {
child, err := nd.Links[i].GetNode(context.TODO(), ds)
if err != nil {
return err
}
if i < direct {
// Direct blocks
err := verifyTDagRec(child, 0, direct, layerRepeat, ds)
if err != nil {
return err
}
} else {
// Recursive trickle dags
rdepth := ((i - direct) / layerRepeat) + 1
if rdepth >= depth && depth > 0 {
return errors.New("Child dag was too deep!")
}
err := verifyTDagRec(child, rdepth, direct, layerRepeat, ds)
if err != nil {
return err
}
}
}
return nil
}
func (ids *IDService) IdentifyConn(c inet.Conn) {
ids.currmu.Lock()
if wait, found := ids.currid[c]; found {
ids.currmu.Unlock()
log.Debugf("IdentifyConn called twice on: %s", c)
<-wait // already identifying it. wait for it.
return
}
ch := make(chan struct{})
ids.currid[c] = ch
ids.currmu.Unlock()
defer close(ch)
s, err := c.NewStream()
if err != nil {
log.Debugf("error opening initial stream for %s: %s", ID, err)
log.Event(context.TODO(), "IdentifyOpenFailed", c.RemotePeer())
c.Close()
return
}
bwc := ids.Host.GetBandwidthReporter()
s = mstream.WrapStream(s, ID, bwc)
// ok give the response to our handler.
if err := msmux.SelectProtoOrFail(ID, s); err != nil {
log.Debugf("error writing stream header for %s", ID)
log.Event(context.TODO(), "IdentifyOpenFailed", c.RemotePeer())
s.Close()
return
}
ids.ResponseHandler(s)
ids.currmu.Lock()
_, found := ids.currid[c]
delete(ids.currid, c)
ids.currmu.Unlock()
if !found {
log.Debugf("IdentifyConn failed to find channel (programmer error) for %s", c)
return
}
}
func catNode(ds dag.DAGService, nd *dag.Node) ([]byte, error) {
r, err := uio.NewDagReader(context.TODO(), nd, ds)
if err != nil {
return nil, err
}
defer r.Close()
return ioutil.ReadAll(r)
}
func logProtocolMismatchDisconnect(c inet.Conn, protocol, agent string) {
lm := make(lgbl.DeferredMap)
lm["remotePeer"] = func() interface{} { return c.RemotePeer().Pretty() }
lm["remoteAddr"] = func() interface{} { return c.RemoteMultiaddr().String() }
lm["protocolVersion"] = protocol
lm["agentVersion"] = agent
log.Event(context.TODO(), "IdentifyProtocolMismatch", lm)
log.Debug("IdentifyProtocolMismatch %s %s %s (disconnected)", c.RemotePeer(), protocol, agent)
}
func (w *writecache) Put(b blocks.Block) error {
k := b.Key()
if _, ok := w.cache.Get(k); ok {
return nil
}
defer log.EventBegin(context.TODO(), "writecache.BlockAdded", &k).Done()
w.cache.Add(b.Key(), struct{}{})
return w.blockstore.Put(b)
}
func (n *network) deliver(
r bsnet.Receiver, from peer.ID, message bsmsg.BitSwapMessage) error {
if message == nil || from == "" {
return errors.New("Invalid input")
}
n.delay.Wait()
r.ReceiveMessage(context.TODO(), from, message)
return nil
}
func (w *writecache) PutMany(bs []blocks.Block) error {
var good []blocks.Block
for _, b := range bs {
if _, ok := w.cache.Get(b.Key()); !ok {
good = append(good, b)
k := b.Key()
defer log.EventBegin(context.TODO(), "writecache.BlockAdded", &k).Done()
}
}
return w.blockstore.PutMany(good)
}
func testBloomCached(bs GCBlockstore, ctx context.Context) (*bloomcache, error) {
if ctx == nil {
ctx = context.TODO()
}
opts := DefaultCacheOpts()
bbs, err := CachedBlockstore(bs, ctx, opts)
if err == nil {
return bbs.(*bloomcache), nil
} else {
return nil, err
}
}
// LoadPinner loads a pinner and its keysets from the given datastore
func LoadPinner(d ds.Datastore, dserv mdag.DAGService) (Pinner, error) {
p := new(pinner)
rootKeyI, err := d.Get(pinDatastoreKey)
if err != nil {
return nil, fmt.Errorf("cannot load pin state: %v", err)
}
rootKeyBytes, ok := rootKeyI.([]byte)
if !ok {
return nil, fmt.Errorf("cannot load pin state: %s was not bytes", pinDatastoreKey)
}
rootKey := key.Key(rootKeyBytes)
ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5)
defer cancel()
root, err := dserv.Get(ctx, rootKey)
if err != nil {
return nil, fmt.Errorf("cannot find pinning root object: %v", err)
}
internalPin := map[key.Key]struct{}{
rootKey: struct{}{},
}
recordInternal := func(k key.Key) {
internalPin[k] = struct{}{}
}
{ // load recursive set
recurseKeys, err := loadSet(ctx, dserv, root, linkRecursive, recordInternal)
if err != nil {
return nil, fmt.Errorf("cannot load recursive pins: %v", err)
}
p.recursePin = set.SimpleSetFromKeys(recurseKeys)
}
{ // load direct set
directKeys, err := loadSet(ctx, dserv, root, linkDirect, recordInternal)
if err != nil {
return nil, fmt.Errorf("cannot load direct pins: %v", err)
}
p.directPin = set.SimpleSetFromKeys(directKeys)
}
p.internalPin = internalPin
// assign services
p.dserv = dserv
p.dstore = d
return p, nil
}
// Flush encodes and writes pinner keysets to the datastore
func (p *pinner) Flush() error {
p.lock.Lock()
defer p.lock.Unlock()
ctx := context.TODO()
internalPin := make(map[key.Key]struct{})
recordInternal := func(k key.Key) {
internalPin[k] = struct{}{}
}
root := &mdag.Node{}
{
n, err := storeSet(ctx, p.dserv, p.directPin.GetKeys(), recordInternal)
if err != nil {
return err
}
if err := root.AddNodeLink(linkDirect, n); err != nil {
return err
}
}
{
n, err := storeSet(ctx, p.dserv, p.recursePin.GetKeys(), recordInternal)
if err != nil {
return err
}
if err := root.AddNodeLink(linkRecursive, n); err != nil {
return err
}
}
// add the empty node, its referenced by the pin sets but never created
_, err := p.dserv.Add(new(mdag.Node))
if err != nil {
return err
}
k, err := p.dserv.Add(root)
if err != nil {
return err
}
internalPin[k] = struct{}{}
if err := p.dstore.Put(pinDatastoreKey, []byte(k)); err != nil {
return fmt.Errorf("cannot store pin state: %v", err)
}
p.internalPin = internalPin
return nil
}
func addPortMapping(nmgr *natManager, intaddr ma.Multiaddr) {
nat := nmgr.NAT()
if nat == nil {
panic("natManager addPortMapping called without a nat.")
}
// first, check if the port mapping already exists.
for _, mapping := range nat.Mappings() {
if mapping.InternalAddr().Equal(intaddr) {
return // it exists! return.
}
}
ctx := context.TODO()
lm := make(lgbl.DeferredMap)
lm["internalAddr"] = func() interface{} { return intaddr.String() }
defer log.EventBegin(ctx, "natMgrAddPortMappingWait", lm).Done()
select {
case <-nmgr.proc.Closing():
lm["outcome"] = "cancelled"
return // no use.
case <-nmgr.ready: // wait until it's ready.
}
// actually start the port map (sub-event because waiting may take a while)
defer log.EventBegin(ctx, "natMgrAddPortMapping", lm).Done()
// get the nat
m, err := nat.NewMapping(intaddr)
if err != nil {
lm["outcome"] = "failure"
lm["error"] = err
return
}
extaddr, err := m.ExternalAddr()
if err != nil {
lm["outcome"] = "failure"
lm["error"] = err
return
}
lm["outcome"] = "success"
lm["externalAddr"] = func() interface{} { return extaddr.String() }
log.Infof("established nat port mapping: %s <--> %s", intaddr, extaddr)
}
func (lb *Loopback) HandleStream(s inet.Stream) {
defer s.Close()
pbr := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
var incoming dhtpb.Message
if err := pbr.ReadMsg(&incoming); err != nil {
log.Debug(err)
return
}
ctx := context.TODO()
outgoing := lb.Handler.HandleRequest(ctx, s.Conn().RemotePeer(), &incoming)
pbw := ggio.NewDelimitedWriter(s)
if err := pbw.WriteMsg(outgoing); err != nil {
return // TODO logerr
}
}
func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) {
fi.nodelk.Lock()
node := fi.node
fi.nodelk.Unlock()
fsn, err := ft.FSNodeFromBytes(node.Data())
if err != nil {
return nil, err
}
switch fsn.Type {
default:
return nil, fmt.Errorf("unsupported fsnode type for 'file'")
case ft.TSymlink:
return nil, fmt.Errorf("symlinks not yet supported")
case ft.TFile, ft.TRaw:
// OK case
}
switch flags {
case OpenReadOnly:
fi.desclock.RLock()
case OpenWriteOnly, OpenReadWrite:
fi.desclock.Lock()
default:
// TODO: support other modes
return nil, fmt.Errorf("mode not supported")
}
dmod, err := mod.NewDagModifier(context.TODO(), node, fi.dserv, chunk.DefaultSplitter)
if err != nil {
return nil, err
}
return &fileDescriptor{
inode: fi,
perms: flags,
sync: sync,
mod: dmod,
}, nil
}
// getNode returns the node for link. If it return an error,
// stop processing. if it returns a nil node, just skip it.
//
// the error handling is a little complicated.
func (t *traversal) getNode(link *mdag.Link) (*mdag.Node, error) {
getNode := func(l *mdag.Link) (*mdag.Node, error) {
next, err := l.GetNode(context.TODO(), t.opts.DAG)
if err != nil {
return nil, err
}
skip, err := t.shouldSkip(next)
if skip {
next = nil
}
return next, err
}
next, err := getNode(link)
if err != nil && t.opts.ErrFunc != nil { // attempt recovery.
err = t.opts.ErrFunc(err)
next = nil // skip regardless
}
return next, err
}
// ResolveUnspecifiedAddresses expands unspecified ip addresses (/ip4/0.0.0.0, /ip6/::) to
// use the known local interfaces.
func ResolveUnspecifiedAddresses(unspecAddrs, ifaceAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) {
// todo optimize: only fetch these if we have a "any" addr.
if len(ifaceAddrs) < 1 {
var err error
ifaceAddrs, err = InterfaceAddresses()
if err != nil {
return nil, err
}
// log.Debug("InterfaceAddresses:", ifaceAddrs)
}
var outputAddrs []ma.Multiaddr
for _, a := range unspecAddrs {
// unspecified?
resolved, err := ResolveUnspecifiedAddress(a, ifaceAddrs)
if err != nil {
continue // optimistic. if we cant resolve anything, we'll know at the bottom.
}
// log.Debug("resolved:", a, resolved)
outputAddrs = append(outputAddrs, resolved...)
}
if len(outputAddrs) < 1 {
return nil, fmt.Errorf("failed to specify addrs: %s", unspecAddrs)
}
log.Event(context.TODO(), "interfaceListenAddresses", func() logging.Loggable {
var addrs []string
for _, addr := range outputAddrs {
addrs = append(addrs, addr.String())
}
return logging.Metadata{"addresses": addrs}
}())
log.Debug("ResolveUnspecifiedAddresses:", unspecAddrs, ifaceAddrs, outputAddrs)
return outputAddrs, nil
}
// Context returns the IpfsNode context
func (n *IpfsNode) Context() context.Context {
if n.ctx == nil {
n.ctx = context.TODO()
}
return n.ctx
}
func (pm *WantManager) CancelWants(ks []key.Key) {
pm.addEntries(context.TODO(), ks, true)
}
func TestRepublisher(t *testing.T) {
if ci.IsRunning() {
t.Skip("dont run timing tests in CI")
}
ctx := context.TODO()
pub := make(chan struct{})
pf := func(ctx context.Context, k key.Key) error {
pub <- struct{}{}
return nil
}
tshort := time.Millisecond * 50
tlong := time.Second / 2
rp := NewRepublisher(ctx, pf, tshort, tlong)
go rp.Run()
rp.Update("test")
// should hit short timeout
select {
case <-time.After(tshort * 2):
t.Fatal("publish didnt happen in time")
case <-pub:
}
cctx, cancel := context.WithCancel(context.Background())
go func() {
for {
rp.Update("a")
time.Sleep(time.Millisecond * 10)
select {
case <-cctx.Done():
return
default:
}
}
}()
select {
case <-pub:
t.Fatal("shouldnt have received publish yet!")
case <-time.After((tlong * 9) / 10):
}
select {
case <-pub:
case <-time.After(tlong / 2):
t.Fatal("waited too long for pub!")
}
cancel()
go func() {
err := rp.Close()
if err != nil {
t.Fatal(err)
}
}()
// final pub from closing
<-pub
}
func (c *client) Send(req cmds.Request) (cmds.Response, error) {
if req.Context() == nil {
log.Warningf("no context set in request")
if err := req.SetRootContext(context.TODO()); err != nil {
return nil, err
}
}
// save user-provided encoding
previousUserProvidedEncoding, found, err := req.Option(cmds.EncShort).String()
if err != nil {
return nil, err
}
// override with json to send to server
req.SetOption(cmds.EncShort, cmds.JSON)
// stream channel output
req.SetOption(cmds.ChanOpt, "true")
query, err := getQuery(req)
if err != nil {
return nil, err
}
var fileReader *MultiFileReader
var reader io.Reader
if req.Files() != nil {
fileReader = NewMultiFileReader(req.Files(), true)
reader = fileReader
}
path := strings.Join(req.Path(), "/")
url := fmt.Sprintf(ApiUrlFormat, c.serverAddress, ApiPath, path, query)
httpReq, err := http.NewRequest("POST", url, reader)
if err != nil {
return nil, err
}
// TODO extract string consts?
if fileReader != nil {
httpReq.Header.Set(contentTypeHeader, "multipart/form-data; boundary="+fileReader.Boundary())
} else {
httpReq.Header.Set(contentTypeHeader, applicationOctetStream)
}
httpReq.Header.Set(uaHeader, config.ApiVersion)
httpReq.Cancel = req.Context().Done()
httpReq.Close = true
httpRes, err := c.httpClient.Do(httpReq)
if err != nil {
return nil, err
}
// using the overridden JSON encoding in request
res, err := getResponse(httpRes, req)
if err != nil {
return nil, err
}
if found && len(previousUserProvidedEncoding) > 0 {
// reset to user provided encoding after sending request
// NB: if user has provided an encoding but it is the empty string,
// still leave it as JSON.
req.SetOption(cmds.EncShort, previousUserProvidedEncoding)
}
return res, nil
}
func (rs *mocknetserver) Client(p testutil.Identity) Client {
return rs.ClientWithDatastore(context.TODO(), p, ds.NewMapDatastore())
}
func (w *writecache) DeleteBlock(k key.Key) error {
defer log.EventBegin(context.TODO(), "writecache.BlockRemoved", &k).Done()
w.cache.Remove(k)
return w.blockstore.DeleteBlock(k)
}
