// AddWrapped adds data from a reader, and wraps it with a directory object
// to preserve the filename.
// Returns the path of the added file ("<dir hash>/filename"), the DAG node of
// the directory, and and error if any.
func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.Node, error) {
file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil)
fileAdder, err := NewAdder(n.Context(), n, nil)
if err != nil {
return "", nil, err
}
fileAdder.Wrap = true
defer n.Blockstore.PinLock().Unlock()
err = fileAdder.addFile(file)
if err != nil {
return "", nil, err
}
dagnode, err := fileAdder.Finalize()
if err != nil {
return "", nil, err
}
k, err := dagnode.Key()
if err != nil {
return "", nil, err
}
return gopath.Join(k.String(), filename), dagnode, nil
}
// AddWrapped adds data from a reader, and wraps it with a directory object
// to preserve the filename.
// Returns the path of the added file ("<dir hash>/filename"), the DAG node of
// the directory, and and error if any.
func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *merkledag.Node, error) {
file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil)
dir := files.NewSliceFile("", "", []files.File{file})
dagnode, err := addDir(n, dir)
if err != nil {
return "", nil, err
}
k, err := dagnode.Key()
if err != nil {
return "", nil, err
}
return gopath.Join(k.String(), filename), dagnode, nil
}
func testAddWPosInfo(t *testing.T, rawLeaves bool) {
r := &repo.Mock{
C: config.Config{
Identity: config.Identity{
PeerID: "Qmfoo", // required by offline node
},
},
D: testutil.ThreadSafeCloserMapDatastore(),
}
node, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
if err != nil {
t.Fatal(err)
}
bs := &testBlockstore{GCBlockstore: node.Blockstore, expectedPath: "/tmp/foo.txt", t: t}
bserv := blockservice.New(bs, node.Exchange)
dserv := dag.NewDAGService(bserv)
adder, err := NewAdder(context.Background(), node.Pinning, bs, dserv)
if err != nil {
t.Fatal(err)
}
adder.Out = make(chan interface{})
adder.Progress = true
adder.RawLeaves = rawLeaves
data := make([]byte, 5*1024*1024)
rand.New(rand.NewSource(2)).Read(data) // Rand.Read never returns an error
fileData := ioutil.NopCloser(bytes.NewBuffer(data))
fileInfo := dummyFileInfo{"foo.txt", int64(len(data)), time.Now()}
file := files.NewReaderFile("foo.txt", "/tmp/foo.txt", fileData, &fileInfo)
go func() {
defer close(adder.Out)
err = adder.AddFile(file)
if err != nil {
t.Fatal(err)
}
}()
for _ = range adder.Out {
}
if bs.countAtOffsetZero != 2 {
t.Fatal("expected 2 blocks with an offset at zero (one root and one leafh), got", bs.countAtOffsetZero)
}
if bs.countAtOffsetNonZero != 19 {
// note: the exact number will depend on the size and the sharding algo. used
t.Fatal("expected 19 blocks with an offset > 0, got", bs.countAtOffsetNonZero)
}
}
func appendStdinAsFile(args []files.File, stdin *os.File) ([]files.File, *os.File) {
arg := files.NewReaderFile("", "", stdin, nil)
return append(args, arg), nil
}
func TestAddGCLive(t *testing.T) {
r := &repo.Mock{
C: config.Config{
Identity: config.Identity{
PeerID: "Qmfoo", // required by offline node
},
},
D: testutil.ThreadSafeCloserMapDatastore(),
}
node, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
if err != nil {
t.Fatal(err)
}
errs := make(chan error)
out := make(chan interface{})
adder, err := NewAdder(context.Background(), node, out)
if err != nil {
t.Fatal(err)
}
dataa := ioutil.NopCloser(bytes.NewBufferString("testfileA"))
rfa := files.NewReaderFile("a", "a", dataa, nil)
// make two files with pipes so we can 'pause' the add for timing of the test
piper, pipew := io.Pipe()
hangfile := files.NewReaderFile("b", "b", piper, nil)
datad := ioutil.NopCloser(bytes.NewBufferString("testfileD"))
rfd := files.NewReaderFile("d", "d", datad, nil)
slf := files.NewSliceFile("files", "files", []files.File{rfa, hangfile, rfd})
addDone := make(chan struct{})
go func() {
defer close(addDone)
defer close(out)
err := adder.AddFile(slf)
if err != nil {
t.Fatal(err)
}
}()
addedHashes := make(map[string]struct{})
select {
case o := <-out:
addedHashes[o.(*AddedObject).Hash] = struct{}{}
case <-addDone:
t.Fatal("add shouldnt complete yet")
}
var gcout <-chan key.Key
gcstarted := make(chan struct{})
go func() {
defer close(gcstarted)
gcchan, err := gc.GC(context.Background(), node.Blockstore, node.Pinning)
if err != nil {
log.Error("GC ERROR:", err)
errs <- err
return
}
gcout = gcchan
}()
// gc shouldnt start until we let the add finish its current file.
pipew.Write([]byte("some data for file b"))
select {
case <-gcstarted:
t.Fatal("gc shouldnt have started yet")
case err := <-errs:
t.Fatal(err)
default:
}
time.Sleep(time.Millisecond * 100) // make sure gc gets to requesting lock
// finish write and unblock gc
pipew.Close()
// receive next object from adder
select {
case o := <-out:
addedHashes[o.(*AddedObject).Hash] = struct{}{}
case err := <-errs:
t.Fatal(err)
}
select {
case <-gcstarted:
case err := <-errs:
t.Fatal(err)
}
for k := range gcout {
if _, ok := addedHashes[k.B58String()]; ok {
t.Fatal("gc'ed a hash we just added")
}
}
var last key.Key
for a := range out {
// wait for it to finish
last = key.B58KeyDecode(a.(*AddedObject).Hash)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
root, err := node.DAG.Get(ctx, last)
if err != nil {
t.Fatal(err)
}
err = dag.EnumerateChildren(ctx, node.DAG, root, key.NewKeySet())
if err != nil {
t.Fatal(err)
}
}
func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive, hidden bool, root *cmds.Command) ([]string, []files.File, error) {
// ignore stdin on Windows
if runtime.GOOS == "windows" {
stdin = nil
}
// check if stdin is coming from terminal or is being piped in
if stdin != nil {
if term, err := isTerminal(stdin); err != nil {
return nil, nil, err
} else if term {
stdin = nil // set to nil so we ignore it
}
}
// count required argument definitions
numRequired := 0
for _, argDef := range argDefs {
if argDef.Required {
numRequired++
}
}
// count number of values provided by user.
// if there is at least one ArgDef, we can safely trigger the inputs loop
// below to parse stdin.
numInputs := len(inputs)
if len(argDefs) > 0 && argDefs[len(argDefs)-1].SupportsStdin && stdin != nil {
numInputs += 1
}
// if we have more arg values provided than argument definitions,
// and the last arg definition is not variadic (or there are no definitions), return an error
notVariadic := len(argDefs) == 0 || !argDefs[len(argDefs)-1].Variadic
if notVariadic && len(inputs) > len(argDefs) {
suggestions := suggestUnknownCmd(inputs, root)
if len(suggestions) > 1 {
return nil, nil, fmt.Errorf("Unknown Command \"%s\"\n\nDid you mean any of these?\n\n\t%s", inputs[0], strings.Join(suggestions, "\n\t"))
} else if len(suggestions) > 0 {
return nil, nil, fmt.Errorf("Unknown Command \"%s\"\n\nDid you mean this?\n\n\t%s", inputs[0], suggestions[0])
} else {
return nil, nil, fmt.Errorf("Unknown Command \"%s\"\n", inputs[0])
}
}
stringArgs := make([]string, 0, numInputs)
fileArgs := make(map[string]files.File)
argDefIndex := 0 // the index of the current argument definition
for i := 0; i < numInputs; i++ {
argDef := getArgDef(argDefIndex, argDefs)
// skip optional argument definitions if there aren't sufficient remaining inputs
for numInputs-i <= numRequired && !argDef.Required {
argDefIndex++
argDef = getArgDef(argDefIndex, argDefs)
}
if argDef.Required {
numRequired--
}
var err error
if argDef.Type == cmds.ArgString {
if stdin == nil || !argDef.SupportsStdin {
// add string values
stringArgs, inputs = appendString(stringArgs, inputs)
} else {
if len(inputs) > 0 {
// don't use stdin if we have inputs
stdin = nil
} else {
// if we have a stdin, read it in and use the data as a string value
stringArgs, stdin, err = appendStdinAsString(stringArgs, stdin)
if err != nil {
return nil, nil, err
}
}
}
} else if argDef.Type == cmds.ArgFile {
if stdin == nil || !argDef.SupportsStdin {
// treat stringArg values as file paths
fpath := inputs[0]
inputs = inputs[1:]
file, err := appendFile(fpath, argDef, recursive, hidden)
if err != nil {
return nil, nil, err
}
fileArgs[fpath] = file
} else {
if len(inputs) > 0 {
// don't use stdin if we have inputs
stdin = nil
} else {
// if we have a stdin, create a file from it
fileArgs[""] = files.NewReaderFile("", "", stdin, nil)
}
}
}
argDefIndex++
}
// check to make sure we didn't miss any required arguments
if len(argDefs) > argDefIndex {
for _, argDef := range argDefs[argDefIndex:] {
if argDef.Required {
return nil, nil, fmt.Errorf("Argument '%s' is required", argDef.Name)
}
}
}
return stringArgs, filesMapToSortedArr(fileArgs), nil
}
func TestOutput(t *testing.T) {
text := "Some text! :)"
fileset := []files.File{
files.NewReaderFile("file.txt", ioutil.NopCloser(strings.NewReader(text)), nil),
files.NewSliceFile("boop", []files.File{
files.NewReaderFile("boop/a.txt", ioutil.NopCloser(strings.NewReader("bleep")), nil),
files.NewReaderFile("boop/b.txt", ioutil.NopCloser(strings.NewReader("bloop")), nil),
}),
files.NewReaderFile("beep.txt", ioutil.NopCloser(strings.NewReader("beep")), nil),
}
sf := files.NewSliceFile("", fileset)
buf := make([]byte, 20)
// testing output by reading it with the go stdlib "mime/multipart" Reader
mfr := NewMultiFileReader(sf, true)
mpReader := multipart.NewReader(mfr, mfr.Boundary())
part, err := mpReader.NextPart()
if part == nil || err != nil {
t.Error("Expected non-nil part, nil error")
}
mpf, err := files.NewFileFromPart(part)
if mpf == nil || err != nil {
t.Error("Expected non-nil MultipartFile, nil error")
}
if mpf.IsDirectory() {
t.Error("Expected file to not be a directory")
}
if mpf.FileName() != "file.txt" {
t.Error("Expected filename to be \"file.txt\"")
}
if n, err := mpf.Read(buf); n != len(text) || err != nil {
t.Error("Expected to read from file", n, err)
}
if string(buf[:len(text)]) != text {
t.Error("Data read was different than expected")
}
part, err = mpReader.NextPart()
if part == nil || err != nil {
t.Error("Expected non-nil part, nil error")
}
mpf, err = files.NewFileFromPart(part)
if mpf == nil || err != nil {
t.Error("Expected non-nil MultipartFile, nil error")
}
if !mpf.IsDirectory() {
t.Error("Expected file to be a directory")
}
if mpf.FileName() != "boop" {
t.Error("Expected filename to be \"boop\"")
}
child, err := mpf.NextFile()
if child == nil || err != nil {
t.Error("Expected to be able to read a child file")
}
if child.IsDirectory() {
t.Error("Expected file to not be a directory")
}
if child.FileName() != "boop/a.txt" {
t.Error("Expected filename to be \"some/file/path\"")
}
child, err = mpf.NextFile()
if child == nil || err != nil {
t.Error("Expected to be able to read a child file")
}
if child.IsDirectory() {
t.Error("Expected file to not be a directory")
}
if child.FileName() != "boop/b.txt" {
t.Error("Expected filename to be \"some/file/path\"")
}
child, err = mpf.NextFile()
if child != nil || err != io.EOF {
t.Error("Expected to get (nil, io.EOF)")
}
part, err = mpReader.NextPart()
if part == nil || err != nil {
t.Error("Expected non-nil part, nil error")
}
mpf, err = files.NewFileFromPart(part)
if mpf == nil || err != nil {
t.Error("Expected non-nil MultipartFile, nil error")
}
part, err = mpReader.NextPart()
if part != nil || err != io.EOF {
t.Error("Expected to get (nil, io.EOF)")
}
}
func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive, hidden bool, root *cmds.Command) ([]string, []files.File, error) {
// ignore stdin on Windows
if runtime.GOOS == "windows" {
stdin = nil
}
// count required argument definitions
numRequired := 0
for _, argDef := range argDefs {
if argDef.Required {
numRequired++
}
}
// count number of values provided by user.
// if there is at least one ArgDef, we can safely trigger the inputs loop
// below to parse stdin.
numInputs := len(inputs)
if len(argDefs) > 0 && argDefs[len(argDefs)-1].SupportsStdin && stdin != nil {
numInputs += 1
}
// if we have more arg values provided than argument definitions,
// and the last arg definition is not variadic (or there are no definitions), return an error
notVariadic := len(argDefs) == 0 || !argDefs[len(argDefs)-1].Variadic
if notVariadic && len(inputs) > len(argDefs) {
err := printSuggestions(inputs, root)
return nil, nil, err
}
stringArgs := make([]string, 0, numInputs)
fileArgs := make(map[string]files.File)
argDefIndex := 0 // the index of the current argument definition
for i := 0; i < numInputs; i++ {
argDef := getArgDef(argDefIndex, argDefs)
// skip optional argument definitions if there aren't sufficient remaining inputs
for numInputs-i <= numRequired && !argDef.Required {
argDefIndex++
argDef = getArgDef(argDefIndex, argDefs)
}
if argDef.Required {
numRequired--
}
fillingVariadic := argDefIndex+1 > len(argDefs)
switch argDef.Type {
case cmds.ArgString:
if len(inputs) > 0 {
stringArgs, inputs = append(stringArgs, inputs[0]), inputs[1:]
} else if stdin != nil && argDef.SupportsStdin && !fillingVariadic {
if r, err := maybeWrapStdin(stdin, msgStdinInfo); err == nil {
fileArgs[stdin.Name()] = files.NewReaderFile("stdin", "", r, nil)
stdin = nil
}
}
case cmds.ArgFile:
if len(inputs) > 0 {
// treat stringArg values as file paths
fpath := inputs[0]
inputs = inputs[1:]
var file files.File
if fpath == "-" {
r, err := maybeWrapStdin(stdin, msgStdinInfo)
if err != nil {
return nil, nil, err
}
fpath = stdin.Name()
file = files.NewReaderFile("", fpath, r, nil)
} else {
nf, err := appendFile(fpath, argDef, recursive, hidden)
if err != nil {
return nil, nil, err
}
file = nf
}
fileArgs[fpath] = file
} else if stdin != nil && argDef.SupportsStdin &&
argDef.Required && !fillingVariadic {
r, err := maybeWrapStdin(stdin, msgStdinInfo)
if err != nil {
return nil, nil, err
}
fpath := stdin.Name()
fileArgs[fpath] = files.NewReaderFile("", fpath, r, nil)
}
}
argDefIndex++
}
// check to make sure we didn't miss any required arguments
if len(argDefs) > argDefIndex {
for _, argDef := range argDefs[argDefIndex:] {
if argDef.Required {
return nil, nil, fmt.Errorf("Argument '%s' is required", argDef.Name)
}
}
}
return stringArgs, filesMapToSortedArr(fileArgs), nil
}
