func DirectAddCat(data []byte, conf testutil.LatencyConfig) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
const numPeers = 2
// create network
mn, err := mocknet.FullMeshLinked(ctx, numPeers)
if err != nil {
return err
}
mn.SetLinkDefaults(mocknet.LinkOptions{
Latency: conf.NetworkLatency,
// TODO add to conf. This is tricky because we want 0 values to be functional.
Bandwidth: math.MaxInt32,
})
peers := mn.Peers()
if len(peers) < numPeers {
return errors.New("test initialization error")
}
adder, err := core.NewIPFSNode(ctx, core.ConfigOption(MocknetTestRepo(peers[0], mn.Host(peers[0]), conf, core.DHTOption)))
if err != nil {
return err
}
defer adder.Close()
catter, err := core.NewIPFSNode(ctx, core.ConfigOption(MocknetTestRepo(peers[1], mn.Host(peers[1]), conf, core.DHTOption)))
if err != nil {
return err
}
defer catter.Close()
bs1 := []peer.PeerInfo{adder.Peerstore.PeerInfo(adder.Identity)}
bs2 := []peer.PeerInfo{catter.Peerstore.PeerInfo(catter.Identity)}
if err := catter.Bootstrap(core.BootstrapConfigWithPeers(bs1)); err != nil {
return err
}
if err := adder.Bootstrap(core.BootstrapConfigWithPeers(bs2)); err != nil {
return err
}
added, err := coreunix.Add(adder, bytes.NewReader(data))
if err != nil {
return err
}
readerCatted, err := coreunix.Cat(catter, added)
if err != nil {
return err
}
// verify
bufout := new(bytes.Buffer)
io.Copy(bufout, readerCatted)
if 0 != bytes.Compare(bufout.Bytes(), data) {
return errors.New("catted data does not match added data")
}
return nil
}
func TestCatBasic(t *testing.T) {
ctx := context.Background()
node, api, err := makeAPI(ctx)
if err != nil {
t.Fatal(err)
}
hr := strings.NewReader(helloStr)
k, err := coreunix.Add(node, hr)
if err != nil {
t.Fatal(err)
}
if k != hello {
t.Fatalf("expected CID %s, got: %s", hello, k)
}
r, err := api.Cat(ctx, k)
if err != nil {
t.Fatal(err)
}
buf := make([]byte, len(helloStr))
_, err = io.ReadFull(r, buf)
if err != nil {
t.Error(err)
}
if string(buf) != helloStr {
t.Fatalf("expected [%s], got [%s] [err=%s]", helloStr, string(buf), err)
}
}
func runFileAddingWorker(n *core.IpfsNode) error {
errs := make(chan error)
go func() {
var i int64
for i = 1; i < math.MaxInt32; i++ {
piper, pipew := io.Pipe()
go func() {
defer pipew.Close()
if err := random.WritePseudoRandomBytes(sizeOfIthFile(i), pipew, *seed); err != nil {
errs <- err
}
}()
k, err := coreunix.Add(n, piper)
if err != nil {
errs <- err
}
log.Println("added file", "seed", *seed, "#", i, "key", k, "size", unit.Information(sizeOfIthFile(i)))
time.Sleep(1 * time.Second)
}
}()
var i int64
for i = 0; i < math.MaxInt32; i++ {
err := <-errs
if err != nil {
log.Fatal(err)
}
}
return nil
}
func TestCatEmptyFile(t *testing.T) {
ctx := context.Background()
node, api, err := makeAPI(ctx)
if err != nil {
t.Fatal(err)
}
_, err = coreunix.Add(node, strings.NewReader(""))
if err != nil {
t.Fatal(err)
}
r, err := api.Cat(ctx, emptyUnixfsFile)
if err != nil {
t.Fatal(err)
}
buf := make([]byte, 1) // non-zero so that Read() actually tries to read
n, err := io.ReadFull(r, buf)
if err != nil && err != io.EOF {
t.Error(err)
}
if !bytes.HasPrefix(buf, []byte{0x00}) {
t.Fatalf("expected empty data, got [%s] [read=%d]", buf, n)
}
}
func runFileCattingWorker(ctx context.Context, n *core.IpfsNode) error {
conf, err := config.Init(ioutil.Discard, *nBitsForKeypair)
if err != nil {
return err
}
r := &repo.Mock{
D: ds2.CloserWrap(syncds.MutexWrap(datastore.NewMapDatastore())),
C: *conf,
}
dummy, err := core.NewNode(ctx, &core.BuildCfg{
Repo: r,
})
if err != nil {
return err
}
errs := make(chan error)
go func() {
defer dummy.Close()
var i int64 = 1
for {
buf := new(bytes.Buffer)
if err := random.WritePseudoRandomBytes(sizeOfIthFile(i), buf, *seed); err != nil {
errs <- err
}
// add to a dummy node to discover the key
k, err := coreunix.Add(dummy, bytes.NewReader(buf.Bytes()))
if err != nil {
errs <- err
}
e := elog.EventBegin(ctx, "cat", logging.LoggableF(func() map[string]interface{} {
return map[string]interface{}{
"key": k,
"localPeer": n.Identity,
}
}))
if r, err := coreunix.Cat(ctx, n, k); err != nil {
e.Done()
log.Printf("failed to cat file. seed: %d #%d key: %s err: %s", *seed, i, k, err)
} else {
log.Println("found file", "seed", *seed, "#", i, "key", k, "size", unit.Information(sizeOfIthFile(i)))
io.Copy(ioutil.Discard, r)
e.Done()
log.Println("catted file", "seed", *seed, "#", i, "key", k, "size", unit.Information(sizeOfIthFile(i)))
i++
}
time.Sleep(time.Second)
}
}()
err = <-errs
if err != nil {
log.Fatal(err)
}
return nil
}
func handleAdd(n *core.IpfsNode, ctx context.Context, index *Index, mtx *sync.Mutex, wg *sync.WaitGroup, dspath string, reloadindex chan *Entry, getCurIndex func() *Index) {
list, err := corenet.Listen(n, "/pack/add")
if err != nil {
panic(err)
}
//fmt.Printf("I am ready to add: %s\n", n.Identity)
for {
fmt.Println("Waiting for add...\n")
con, err := list.Accept()
if err != nil {
fmt.Println(err)
return
}
go func() {
defer con.Close()
index = getCurIndex()
fmt.Printf("Connection from: %s\n", con.Conn().RemotePeer())
if len(index.WriteList) != 0 {
present := false
for i := range index.WriteList {
if con.Conn().RemotePeer().Pretty() == index.WriteList[i] {
present = true
break
}
}
if !present {
fmt.Println("Not on write list", con.Conn().RemotePeer().Pretty())
} else {
fmt.Println("On write list", con.Conn().RemotePeer().Pretty())
}
}
serverReader := &serverContentReader{r: con}
fmt.Println("Add request:", serverReader.Name())
key, err := coreunix.Add(n, serverReader)
if err != nil {
panic(err)
}
fmt.Println("Added:", key)
entry := Entry{Timestamp: time.Now(), Size: serverReader.n - HEADER_SIZE, Name: serverReader.Name(), Hash: key}
reloadindex <- &entry
}()
}
wg.Done()
}
func TestGatewayGet(t *testing.T) {
ns := mockNamesys{}
ts, n := newTestServerAndNode(t, ns)
defer ts.Close()
k, err := coreunix.Add(n, strings.NewReader("fnord"))
if err != nil {
t.Fatal(err)
}
ns["/ipns/example.com"] = path.FromString("/ipfs/" + k)
t.Log(ts.URL)
for _, test := range []struct {
host string
path string
status int
text string
}{
{"localhost:5001", "/", http.StatusNotFound, "404 page not found\n"},
{"localhost:5001", "/" + k, http.StatusNotFound, "404 page not found\n"},
{"localhost:5001", "/ipfs/" + k, http.StatusOK, "fnord"},
{"localhost:5001", "/ipns/nxdomain.example.com", http.StatusBadRequest, "Path Resolve error: " + namesys.ErrResolveFailed.Error()},
{"localhost:5001", "/ipns/example.com", http.StatusOK, "fnord"},
{"example.com", "/", http.StatusOK, "fnord"},
} {
var c http.Client
r, err := http.NewRequest("GET", ts.URL+test.path, nil)
if err != nil {
t.Fatal(err)
}
r.Host = test.host
resp, err := c.Do(r)
urlstr := "http://" + test.host + test.path
if err != nil {
t.Errorf("error requesting %s: %s", urlstr, err)
continue
}
defer resp.Body.Close()
if resp.StatusCode != test.status {
t.Errorf("got %d, expected %d from %s", resp.StatusCode, test.status, urlstr)
continue
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("error reading response from %s: %s", urlstr, err)
}
if string(body) != test.text {
t.Errorf("unexpected response body from %s: expected %q; got %q", urlstr, test.text, body)
continue
}
}
}
func addDefaultAssets(out io.Writer, repoRoot string) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
r, err := fsrepo.Open(repoRoot)
if err != nil { // NB: repo is owned by the node
return err
}
nd, err := core.NewIPFSNode(ctx, core.Offline(r))
if err != nil {
return err
}
defer nd.Close()
dirb := uio.NewDirectory(nd.DAG)
// add every file in the assets pkg
for fname, file := range assets.Init_dir {
buf := bytes.NewBufferString(file)
s, err := coreunix.Add(nd, buf)
if err != nil {
return err
}
k := key.B58KeyDecode(s)
if err := dirb.AddChild(fname, k); err != nil {
return err
}
}
dir := dirb.GetNode()
dkey, err := nd.DAG.Add(dir)
if err != nil {
return err
}
if err := nd.Pinning.Pin(ctx, dir, true); err != nil {
return err
}
if err := nd.Pinning.Flush(); err != nil {
return err
}
if _, err = fmt.Fprintf(out, "to get started, enter:\n"); err != nil {
return err
}
_, err = fmt.Fprintf(out, "\n\tipfs cat /ipfs/%s/readme\n\n", dkey)
return err
}
// Add reads `r` and adds it to ipfs.
// The resulting content hash is returned.
func Add(node *Node, r io.Reader) (multihash.Multihash, error) {
nd, err := node.proc()
if err != nil {
return nil, err
}
hash, err := coreunix.Add(nd, r)
if err != nil {
log.Warningf("ipfs add: %v", err)
return nil, err
}
return multihash.FromB58String(hash)
}
func addAssetList(nd *core.IpfsNode, l []string) (*cid.Cid, error) {
dirb := uio.NewDirectory(nd.DAG)
for _, p := range l {
d, err := Asset(p)
if err != nil {
return nil, fmt.Errorf("assets: could load Asset '%s': %s", p, err)
}
s, err := coreunix.Add(nd, bytes.NewBuffer(d))
if err != nil {
return nil, fmt.Errorf("assets: could not Add '%s': %s", p, err)
}
fname := filepath.Base(p)
c, err := cid.Decode(s)
if err != nil {
return nil, err
}
if err := dirb.AddChild(nd.Context(), fname, c); err != nil {
return nil, fmt.Errorf("assets: could not add '%s' as a child: %s", fname, err)
}
}
dir := dirb.GetNode()
dcid, err := nd.DAG.Add(dir)
if err != nil {
return nil, fmt.Errorf("assets: DAG.Add(dir) failed: %s", err)
}
if err := nd.Pinning.Pin(nd.Context(), dir, true); err != nil {
return nil, fmt.Errorf("assets: Pinning on init-docu failed: %s", err)
}
if err := nd.Pinning.Flush(); err != nil {
return nil, fmt.Errorf("assets: Pinning flush failed: %s", err)
}
return dcid, nil
}
func RunSupernodeBootstrappedAddCat(data []byte, conf testutil.LatencyConfig) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
servers, clients, err := InitializeSupernodeNetwork(ctx, 8, 2, conf)
if err != nil {
return err
}
for _, n := range append(servers, clients...) {
defer n.Close()
}
adder := clients[0]
catter := clients[1]
log.Info("adder is", adder.Identity)
log.Info("catter is", catter.Identity)
keyAdded, err := coreunix.Add(adder, bytes.NewReader(data))
if err != nil {
return err
}
readerCatted, err := coreunix.Cat(ctx, catter, keyAdded)
if err != nil {
return err
}
// verify
bufout := new(bytes.Buffer)
io.Copy(bufout, readerCatted)
if 0 != bytes.Compare(bufout.Bytes(), data) {
return errors.New("catted data does not match added data")
}
cancel()
return nil
}
func run(ipfsPath, watchPath string) error {
proc := process.WithParent(process.Background())
log.Printf("running IPFSWatch on '%s' using repo at '%s'...", watchPath, ipfsPath)
ipfsPath, err := homedir.Expand(ipfsPath)
if err != nil {
return err
}
watcher, err := fsnotify.NewWatcher()
if err != nil {
return err
}
defer watcher.Close()
if err := addTree(watcher, watchPath); err != nil {
return err
}
r, err := fsrepo.Open(ipfsPath)
if err != nil {
// TODO handle case: daemon running
// TODO handle case: repo doesn't exist or isn't initialized
return err
}
node, err := core.NewIPFSNode(context.Background(), core.Online(r))
if err != nil {
return err
}
defer node.Close()
if *http {
addr := "/ip4/127.0.0.1/tcp/5001"
var opts = []corehttp.ServeOption{
corehttp.GatewayOption(true),
corehttp.WebUIOption,
corehttp.CommandsOption(cmdCtx(node, ipfsPath)),
}
proc.Go(func(p process.Process) {
if err := corehttp.ListenAndServe(node, addr, opts...); err != nil {
return
}
})
}
interrupts := make(chan os.Signal)
signal.Notify(interrupts, os.Interrupt, os.Kill)
for {
select {
case <-interrupts:
return nil
case e := <-watcher.Events:
log.Printf("received event: %s", e)
isDir, err := IsDirectory(e.Name)
if err != nil {
continue
}
switch e.Op {
case fsnotify.Remove:
if isDir {
if err := watcher.Remove(e.Name); err != nil {
return err
}
}
default:
// all events except for Remove result in an IPFS.Add, but only
// directory creation triggers a new watch
switch e.Op {
case fsnotify.Create:
if isDir {
addTree(watcher, e.Name)
}
}
proc.Go(func(p process.Process) {
file, err := os.Open(e.Name)
if err != nil {
log.Println(err)
}
defer file.Close()
k, err := coreunix.Add(node, file)
if err != nil {
log.Println(err)
}
log.Printf("added %s... key: %s", e.Name, k)
})
}
case err := <-watcher.Errors:
log.Println(err)
}
}
return nil
}
func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
const numPeers = 3
// create network
mn := mocknet.New(ctx)
mn.SetLinkDefaults(mocknet.LinkOptions{
Latency: conf.NetworkLatency,
// TODO add to conf. This is tricky because we want 0 values to be functional.
Bandwidth: math.MaxInt32,
})
bootstrap, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer bootstrap.Close()
adder, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer adder.Close()
catter, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer catter.Close()
mn.LinkAll()
bis := bootstrap.Peerstore.PeerInfo(bootstrap.PeerHost.ID())
bcfg := core.BootstrapConfigWithPeers([]peer.PeerInfo{bis})
if err := adder.Bootstrap(bcfg); err != nil {
return err
}
if err := catter.Bootstrap(bcfg); err != nil {
return err
}
added, err := coreunix.Add(adder, bytes.NewReader(data))
if err != nil {
return err
}
readerCatted, err := coreunix.Cat(ctx, catter, added)
if err != nil {
return err
}
// verify
bufout := new(bytes.Buffer)
io.Copy(bufout, readerCatted)
if 0 != bytes.Compare(bufout.Bytes(), data) {
return errors.New("catted data does not match added data")
}
return nil
}
func benchCat(b *testing.B, data []byte, conf testutil.LatencyConfig) error {
b.StopTimer()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create network
mn := mocknet.New(ctx)
mn.SetLinkDefaults(mocknet.LinkOptions{
Latency: conf.NetworkLatency,
// TODO add to conf. This is tricky because we want 0 values to be functional.
Bandwidth: math.MaxInt32,
})
adder, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer adder.Close()
catter, err := core.NewNode(ctx, &core.BuildCfg{
Online: true,
Host: mock.MockHostOption(mn),
})
if err != nil {
return err
}
defer catter.Close()
err = mn.LinkAll()
if err != nil {
return err
}
bs1 := []peer.PeerInfo{adder.Peerstore.PeerInfo(adder.Identity)}
bs2 := []peer.PeerInfo{catter.Peerstore.PeerInfo(catter.Identity)}
if err := catter.Bootstrap(core.BootstrapConfigWithPeers(bs1)); err != nil {
return err
}
if err := adder.Bootstrap(core.BootstrapConfigWithPeers(bs2)); err != nil {
return err
}
added, err := coreunix.Add(adder, bytes.NewReader(data))
if err != nil {
return err
}
b.StartTimer()
readerCatted, err := coreunix.Cat(ctx, catter, added)
if err != nil {
return err
}
// verify
bufout := new(bytes.Buffer)
io.Copy(bufout, readerCatted)
if 0 != bytes.Compare(bufout.Bytes(), data) {
return errors.New("catted data does not match added data")
}
return nil
}
func TestGatewayGet(t *testing.T) {
t.Skip("not sure whats going on here")
ns := mockNamesys{}
n := newNodeWithMockNamesys(t, ns)
k, err := coreunix.Add(n, strings.NewReader("fnord"))
if err != nil {
t.Fatal(err)
}
ns["example.com"] = path.FromString("/ipfs/" + k)
// need this variable here since we need to construct handler with
// listener, and server with handler. yay cycles.
dh := &delegatedHandler{}
ts := httptest.NewServer(dh)
defer ts.Close()
dh.Handler, err = makeHandler(n,
ts.Listener,
IPNSHostnameOption(),
GatewayOption(false),
)
if err != nil {
t.Fatal(err)
}
t.Log(ts.URL)
for _, test := range []struct {
host string
path string
status int
text string
}{
{"localhost:5001", "/", http.StatusNotFound, "404 page not found\n"},
{"localhost:5001", "/" + k, http.StatusNotFound, "404 page not found\n"},
{"localhost:5001", "/ipfs/" + k, http.StatusOK, "fnord"},
{"localhost:5001", "/ipns/nxdomain.example.com", http.StatusBadRequest, "Path Resolve error: " + namesys.ErrResolveFailed.Error()},
{"localhost:5001", "/ipns/example.com", http.StatusOK, "fnord"},
{"example.com", "/", http.StatusOK, "fnord"},
} {
var c http.Client
r, err := http.NewRequest("GET", ts.URL+test.path, nil)
if err != nil {
t.Fatal(err)
}
r.Host = test.host
resp, err := c.Do(r)
urlstr := "http://" + test.host + test.path
if err != nil {
t.Errorf("error requesting %s: %s", urlstr, err)
continue
}
defer resp.Body.Close()
if resp.StatusCode != test.status {
t.Errorf("got %d, expected %d from %s", resp.StatusCode, test.status, urlstr)
continue
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("error reading response from %s: %s", urlstr, err)
}
if string(body) != test.text {
t.Errorf("unexpected response body from %s: expected %q; got %q", urlstr, test.text, body)
continue
}
}
}