// Untar files from input channel and pass the result to the output channel.
func Untar(opt Options) gonzo.Stage {
return func(ctx context.Context, in <-chan gonzo.File, out chan<- gonzo.File) error {
//Check patterns.
pluck := len(opt.Pluck) > 0
if pluck {
err := match.Good(opt.Pluck...)
if err != nil {
return err
}
}
for {
select {
case file, ok := <-in:
if !ok {
return nil
}
context.WithValue(ctx, "archive", file.FileInfo().Name()).Debug("Untaring")
tr := tar.NewReader(file)
defer file.Close()
// Iterate through the files in the archive.
for {
hdr, err := tr.Next()
if err == io.EOF {
// end of tar archive
break
}
if err != nil {
return err
}
name := strip(opt.StripComponenets, hdr.Name)
if pluck && !match.Any(name, opt.Pluck...) {
continue
}
context.WithValue(ctx, "file", name).Debug("Untaring")
content := new(bytes.Buffer)
n, err := content.ReadFrom(tr)
if err != nil {
return err
}
fs := gonzo.NewFile(ioutil.NopCloser(content), gonzo.FileInfoFrom(hdr.FileInfo()))
fs.FileInfo().SetName(name)
fs.FileInfo().SetSize(int64(n))
out <- fs
}
case <-ctx.Done():
return ctx.Err()
}
}
}
}
// bin is the binary name, it will be passed to os/exec.Command, so the same
// path rules applies.
// the args are the argumetns passed to the program.
func Run(bin string, args ...string) gonzo.Stage {
return func(ctx context.Context, in <-chan gonzo.File, out chan<- gonzo.File) error {
for {
select {
case file, ok := <-in:
if !ok {
return nil
}
cmd := exec.Command(bin, args...)
cmd.Stderr = os.Stderr //TODO: io.Writer logger.
cmd.Stdin = file
ctx = context.WithValue(ctx, "cmd", bin)
ctx.Infof("Passing %s", file.FileInfo().Name())
output, err := cmd.Output()
if err != nil {
return err
}
content := ioutil.NopCloser(bytes.NewReader(output))
out <- gonzo.NewFile(content, file.FileInfo())
case <-ctx.Done():
return ctx.Err()
}
}
}
}
func Put(c Config) gonzo.Stage {
return func(ctx context.Context, files <-chan gonzo.File, out chan<- gonzo.File) error {
err := checkconfig(c)
if err != nil {
return err
}
auth := aws.Auth{
AccessKey: c.AccessKey,
SecretKey: c.SecretKey,
}
con := s3.New(auth, aws.Region(c.Region))
bucket := con.Bucket(c.Name)
for {
select {
case file, ok := <-files:
if !ok {
return nil
}
if file.FileInfo().IsDir() {
continue
}
content, err := ioutil.ReadAll(file)
if err != nil {
return err
}
name := file.FileInfo().Name()
contentType := mime.TypeByExtension(filepath.Ext(name))
if contentType == "" {
contentType = http.DetectContentType(content)
}
ctx = context.WithValue(ctx, "Content-Type", contentType)
ctx.Infof("Uploading %s", name)
err = bucket.Put(name, content, contentType, s3.ACL(c.Perm))
if err != nil {
return err
}
out <- gonzo.NewFile(ioutil.NopCloser(bytes.NewReader(content)), file.FileInfo())
case <-ctx.Done():
return ctx.Err()
}
}
}
}
//Src returns a channel of gonzo.Files that match the provided patterns.
//TODO: ADD support for prefix to avoid all the util.Trims
func Src(ctx context.Context, globs ...string) gonzo.Pipe {
ctx, cancel := context.WithCancel(ctx)
files := make(chan gonzo.File)
pipe := gonzo.NewPipe(ctx, files)
//TODO: Parse globs here, check for invalid globs, split them into "filters".
go func() {
var err error
defer close(files)
fileslist, err := glob.Glob(globs...)
if err != nil {
ctx.Error(err)
return
}
for mp := range fileslist {
var (
file gonzo.File
base = glob.Dir(mp.Glob)
name = mp.Name
)
file, err = Read(mp.Name)
ctx = context.WithValue(ctx, "file", name)
if err == ErrIsDir {
ctx.Warn("fs.Src Ignored Directory.")
continue
}
if err != nil {
cancel()
ctx.Error(err)
return
}
file.FileInfo().SetBase(base)
file.FileInfo().SetName(name)
files <- file
}
}()
return pipe
}
// Unzip the zip files from input channel and pass the result
// to the output channel.
func Unzip() gonzo.Stage {
return func(ctx context.Context, in <-chan gonzo.File, out chan<- gonzo.File) error {
for {
select {
case file, ok := <-in:
if !ok {
return nil
}
raw, err := ioutil.ReadAll(file)
if err != nil {
return err
}
file.Close()
r, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
if err != nil {
return err
}
//counter := c.Counter("unzipping", len(r.File))
// Iterate through the files in the archive,
for _, f := range r.File {
ctx = context.WithValue(ctx, "file", f.Name)
ctx.Info("Unziping")
//counter.Set(i+1, f.Name)
content, err := f.Open()
if err != nil {
}
fs := gonzo.NewFile(content, gonzo.FileInfoFrom(f.FileInfo()))
fs.FileInfo().SetName(f.Name)
out <- fs
}
case <-ctx.Done():
return ctx.Err()
}
}
}
}
// Dest writes the files from the input channel to the dst folder and closes the files.
// It never returns Files.
func Dest(dst string) gonzo.Stage {
return func(ctx context.Context, files <-chan gonzo.File, out chan<- gonzo.File) error {
for {
select {
case file, ok := <-files:
if !ok {
return nil
}
name := file.FileInfo().Name()
path := filepath.Join(dst, filepath.Dir(name))
err := os.MkdirAll(path, 0700)
if err != nil {
return err
}
if file.FileInfo().IsDir() {
out <- file
continue
}
content, err := ioutil.ReadAll(file)
if err != nil {
file.Close()
return err
}
ctx = context.WithValue(ctx, "path", path)
ctx.Infof("Writing %s", name)
err = writeFile(filepath.Join(dst, name), content)
if err != nil {
return err
}
out <- gonzo.NewFile(ioutil.NopCloser(bytes.NewReader(content)), file.FileInfo())
case <-ctx.Done():
return ctx.Err()
}
}
}
}
func get(ctx context.Context, release Release) gonzo.Pipe {
repo := fmt.Sprintf("%s/%s#%s", release.User, release.Repo, release.Tag)
ctx.Warn(repo)
return web.Get(
context.WithValue(ctx, "repo", repo),
fmt.Sprintf(
"https://codeload.github.com/%s/%s/tar.gz/%s",
release.User,
release.Repo,
release.Tag,
),
).Pipe(
gzip.Uncompress(),
tar.Untar(tar.Options{
StripComponenets: 1,
Pluck: release.Pluck,
}),
path.Rename(func(old string) string {
return filepath.Join(release.Repo, old)
}),
)
}
//For The Glory of Debugging.
func List(checkpoint string) gonzo.Stage {
return func(ctx context.Context, in <-chan gonzo.File, out chan<- gonzo.File) error {
ctx = context.WithValue(ctx, "checkpoint", checkpoint)
for {
select {
case file, ok := <-in:
if !ok {
return nil
}
s, err := file.Stat()
if err != nil {
ctx.Error("Can't get File Stat name.")
} else {
ctx.Infof("File %s", s.Name())
}
out <- file
case <-ctx.Done():
return ctx.Err()
}
}
}
}