Beispiel #1
0
// Untar files from input channel and pass the result to the output channel.
func Untar(opt Options) gonzo.Stage {
	return func(ctx context.Context, in <-chan gonzo.File, out chan<- gonzo.File) error {

		//Check patterns.
		pluck := len(opt.Pluck) > 0
		if pluck {
			err := match.Good(opt.Pluck...)
			if err != nil {
				return err
			}
		}

		for {
			select {
			case file, ok := <-in:
				if !ok {
					return nil
				}

				context.WithValue(ctx, "archive", file.FileInfo().Name()).Debug("Untaring")
				tr := tar.NewReader(file)
				defer file.Close()

				// Iterate through the files in the archive.
				for {
					hdr, err := tr.Next()
					if err == io.EOF {
						// end of tar archive
						break
					}
					if err != nil {
						return err
					}

					name := strip(opt.StripComponenets, hdr.Name)
					if pluck && !match.Any(name, opt.Pluck...) {
						continue
					}

					context.WithValue(ctx, "file", name).Debug("Untaring")

					content := new(bytes.Buffer)
					n, err := content.ReadFrom(tr)
					if err != nil {
						return err
					}
					fs := gonzo.NewFile(ioutil.NopCloser(content), gonzo.FileInfoFrom(hdr.FileInfo()))
					fs.FileInfo().SetName(name)
					fs.FileInfo().SetSize(int64(n))

					out <- fs
				}
			case <-ctx.Done():
				return ctx.Err()
			}
		}
	}
}
Beispiel #2
0
// bin is the binary name, it will be passed to os/exec.Command, so the same
// path rules applies.
// the args are the argumetns passed to the program.
func Run(bin string, args ...string) gonzo.Stage {
	return func(ctx context.Context, in <-chan gonzo.File, out chan<- gonzo.File) error {

		for {
			select {
			case file, ok := <-in:
				if !ok {
					return nil
				}

				cmd := exec.Command(bin, args...)
				cmd.Stderr = os.Stderr //TODO: io.Writer logger.
				cmd.Stdin = file

				ctx = context.WithValue(ctx, "cmd", bin)
				ctx.Infof("Passing %s", file.FileInfo().Name())

				output, err := cmd.Output()
				if err != nil {
					return err
				}

				content := ioutil.NopCloser(bytes.NewReader(output))
				out <- gonzo.NewFile(content, file.FileInfo())

			case <-ctx.Done():
				return ctx.Err()
			}
		}
	}
}
Beispiel #3
0
Datei: s3.go Projekt: go-gonzo/s3
func Put(c Config) gonzo.Stage {
	return func(ctx context.Context, files <-chan gonzo.File, out chan<- gonzo.File) error {

		err := checkconfig(c)
		if err != nil {
			return err
		}

		auth := aws.Auth{
			AccessKey: c.AccessKey,
			SecretKey: c.SecretKey,
		}

		con := s3.New(auth, aws.Region(c.Region))
		bucket := con.Bucket(c.Name)

		for {
			select {
			case file, ok := <-files:
				if !ok {
					return nil
				}
				if file.FileInfo().IsDir() {
					continue
				}

				content, err := ioutil.ReadAll(file)
				if err != nil {
					return err
				}

				name := file.FileInfo().Name()

				contentType := mime.TypeByExtension(filepath.Ext(name))
				if contentType == "" {
					contentType = http.DetectContentType(content)
				}
				ctx = context.WithValue(ctx, "Content-Type", contentType)
				ctx.Infof("Uploading %s", name)

				err = bucket.Put(name, content, contentType, s3.ACL(c.Perm))
				if err != nil {
					return err
				}

				out <- gonzo.NewFile(ioutil.NopCloser(bytes.NewReader(content)), file.FileInfo())
			case <-ctx.Done():
				return ctx.Err()
			}
		}
	}
}
Beispiel #4
0
//Src returns a channel of gonzo.Files that match the provided patterns.
//TODO: ADD support for prefix to avoid all the util.Trims
func Src(ctx context.Context, globs ...string) gonzo.Pipe {

	ctx, cancel := context.WithCancel(ctx)

	files := make(chan gonzo.File)
	pipe := gonzo.NewPipe(ctx, files)

	//TODO: Parse globs here, check for invalid globs, split them into "filters".
	go func() {

		var err error
		defer close(files)

		fileslist, err := glob.Glob(globs...)
		if err != nil {
			ctx.Error(err)
			return
		}

		for mp := range fileslist {

			var (
				file gonzo.File
				base = glob.Dir(mp.Glob)
				name = mp.Name
			)

			file, err = Read(mp.Name)
			ctx = context.WithValue(ctx, "file", name)

			if err == ErrIsDir {
				ctx.Warn("fs.Src Ignored Directory.")
				continue

			}

			if err != nil {
				cancel()
				ctx.Error(err)
				return
			}

			file.FileInfo().SetBase(base)
			file.FileInfo().SetName(name)
			files <- file

		}

	}()

	return pipe
}
Beispiel #5
0
// Unzip the zip files from input channel and pass the result
// to the output channel.
func Unzip() gonzo.Stage {
	return func(ctx context.Context, in <-chan gonzo.File, out chan<- gonzo.File) error {

		for {
			select {
			case file, ok := <-in:
				if !ok {
					return nil
				}

				raw, err := ioutil.ReadAll(file)
				if err != nil {
					return err
				}
				file.Close()

				r, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
				if err != nil {
					return err
				}

				//counter := c.Counter("unzipping", len(r.File))

				// Iterate through the files in the archive,
				for _, f := range r.File {
					ctx = context.WithValue(ctx, "file", f.Name)
					ctx.Info("Unziping")
					//counter.Set(i+1, f.Name)

					content, err := f.Open()
					if err != nil {
					}
					fs := gonzo.NewFile(content, gonzo.FileInfoFrom(f.FileInfo()))
					fs.FileInfo().SetName(f.Name)

					out <- fs
				}
			case <-ctx.Done():
				return ctx.Err()
			}
		}
	}
}
Beispiel #6
0
// Dest writes the files from the input channel to the dst folder and closes the files.
// It never returns Files.
func Dest(dst string) gonzo.Stage {
	return func(ctx context.Context, files <-chan gonzo.File, out chan<- gonzo.File) error {

		for {
			select {
			case file, ok := <-files:
				if !ok {
					return nil
				}

				name := file.FileInfo().Name()
				path := filepath.Join(dst, filepath.Dir(name))
				err := os.MkdirAll(path, 0700)
				if err != nil {
					return err
				}

				if file.FileInfo().IsDir() {
					out <- file
					continue
				}

				content, err := ioutil.ReadAll(file)
				if err != nil {
					file.Close()
					return err
				}

				ctx = context.WithValue(ctx, "path", path)
				ctx.Infof("Writing %s", name)
				err = writeFile(filepath.Join(dst, name), content)
				if err != nil {
					return err
				}

				out <- gonzo.NewFile(ioutil.NopCloser(bytes.NewReader(content)), file.FileInfo())

			case <-ctx.Done():
				return ctx.Err()
			}
		}
	}
}
Beispiel #7
0
func get(ctx context.Context, release Release) gonzo.Pipe {

	repo := fmt.Sprintf("%s/%s#%s", release.User, release.Repo, release.Tag)
	ctx.Warn(repo)
	return web.Get(
		context.WithValue(ctx, "repo", repo),
		fmt.Sprintf(
			"https://codeload.github.com/%s/%s/tar.gz/%s",
			release.User,
			release.Repo,
			release.Tag,
		),
	).Pipe(
		gzip.Uncompress(),
		tar.Untar(tar.Options{
			StripComponenets: 1,
			Pluck:            release.Pluck,
		}),
		path.Rename(func(old string) string {
			return filepath.Join(release.Repo, old)
		}),
	)
}
Beispiel #8
0
//For The Glory of Debugging.
func List(checkpoint string) gonzo.Stage {
	return func(ctx context.Context, in <-chan gonzo.File, out chan<- gonzo.File) error {
		ctx = context.WithValue(ctx, "checkpoint", checkpoint)

		for {
			select {
			case file, ok := <-in:
				if !ok {
					return nil
				}
				s, err := file.Stat()
				if err != nil {
					ctx.Error("Can't get File Stat name.")
				} else {
					ctx.Infof("File %s", s.Name())
				}
				out <- file

			case <-ctx.Done():
				return ctx.Err()
			}
		}
	}
}