// Concatenates all the files from the input channel // and passes them to output channel with the given name. func Concat(ctx context.Context, name string) gonzo.Stage { return func(ctx context.Context, files <-chan gonzo.File, out chan<- gonzo.File) error { var ( size int64 bigfile = new(bytes.Buffer) ) err := func() error { for { select { case f, ok := <-files: if !ok { return nil } ctx.Infof( "Adding %s to %s", filepath.Join(f.FileInfo().Base(), f.FileInfo().Name()), name, ) n, err := bigfile.ReadFrom(f) if err != nil { return err } bigfile.WriteRune('\n') size += n + 1 f.Close() case <-ctx.Done(): return ctx.Err() } } }() if err != nil { return err } file := gonzo.NewFile(ioutil.NopCloser(bigfile), gonzo.NewFileInfo()) file.FileInfo().SetSize(size) file.FileInfo().SetName(name) out <- file return nil } }
// Gets the list of urls and passes the results to output channel. // It reports the progress to the Context using a ReadProgress proxy. func Get(ctx context.Context, urls ...string) gonzo.Pipe { ctx, cancel := context.WithCancel(ctx) out := make(chan gonzo.File) client := &http.Client{} go func() { defer close(out) for _, url := range urls { if url == "" { ctx.Error("Empty URL.") cancel() return } select { case <-ctx.Done(): ctx.Warn(context.Canceled) return default: ctx.Infof("Downloading %s", url) file, err := get(ctx, client, url) if err != nil { ctx.Error(err) cancel() break } //TODO: Add progress meter. //s, _ := file.Stat() //file.Reader = c.ReadProgress(file.Reader, "Downloading "+file.Path, s.Size()) out <- file } } }() return gonzo.NewPipe(ctx, out) }