// Concatenates all the files from the input channel
// and passes them to output channel with the given name.
func Concat(ctx context.Context, name string) gonzo.Stage {
return func(ctx context.Context, files <-chan gonzo.File, out chan<- gonzo.File) error {
var (
size int64
bigfile = new(bytes.Buffer)
)
err := func() error {
for {
select {
case f, ok := <-files:
if !ok {
return nil
}
ctx.Infof(
"Adding %s to %s",
filepath.Join(f.FileInfo().Base(), f.FileInfo().Name()),
name,
)
n, err := bigfile.ReadFrom(f)
if err != nil {
return err
}
bigfile.WriteRune('\n')
size += n + 1
f.Close()
case <-ctx.Done():
return ctx.Err()
}
}
}()
if err != nil {
return err
}
file := gonzo.NewFile(ioutil.NopCloser(bigfile), gonzo.NewFileInfo())
file.FileInfo().SetSize(size)
file.FileInfo().SetName(name)
out <- file
return nil
}
}
// Gets the list of urls and passes the results to output channel.
// It reports the progress to the Context using a ReadProgress proxy.
func Get(ctx context.Context, urls ...string) gonzo.Pipe {
ctx, cancel := context.WithCancel(ctx)
out := make(chan gonzo.File)
client := &http.Client{}
go func() {
defer close(out)
for _, url := range urls {
if url == "" {
ctx.Error("Empty URL.")
cancel()
return
}
select {
case <-ctx.Done():
ctx.Warn(context.Canceled)
return
default:
ctx.Infof("Downloading %s", url)
file, err := get(ctx, client, url)
if err != nil {
ctx.Error(err)
cancel()
break
}
//TODO: Add progress meter.
//s, _ := file.Stat()
//file.Reader = c.ReadProgress(file.Reader, "Downloading "+file.Path, s.Size())
out <- file
}
}
}()
return gonzo.NewPipe(ctx, out)
}