コード例 #1
0
ファイル: filesystem.go プロジェクト: go-gonzo/fs
//Src returns a channel of gonzo.Files that match the provided patterns.
//TODO: ADD support for prefix to avoid all the util.Trims
func Src(ctx context.Context, globs ...string) gonzo.Pipe {

	ctx, cancel := context.WithCancel(ctx)

	files := make(chan gonzo.File)
	pipe := gonzo.NewPipe(ctx, files)

	//TODO: Parse globs here, check for invalid globs, split them into "filters".
	go func() {

		var err error
		defer close(files)

		fileslist, err := glob.Glob(globs...)
		if err != nil {
			ctx.Error(err)
			return
		}

		for mp := range fileslist {

			var (
				file gonzo.File
				base = glob.Dir(mp.Glob)
				name = mp.Name
			)

			file, err = Read(mp.Name)
			ctx = context.WithValue(ctx, "file", name)

			if err == ErrIsDir {
				ctx.Warn("fs.Src Ignored Directory.")
				continue

			}

			if err != nil {
				cancel()
				ctx.Error(err)
				return
			}

			file.FileInfo().SetBase(base)
			file.FileInfo().SetName(name)
			files <- file

		}

	}()

	return pipe
}
コード例 #2
0
ファイル: pipe.go プロジェクト: go-gonzo/util
// Merges the output of multiple chan of gonzo.File into a pipe in a serial manner.
// (i.e Reads first chan until the end and moves to the next until the last channel is finished.
func Queue(pipe gonzo.Pipe, pipes ...gonzo.Pipe) gonzo.Pipe {

	if len(pipes) == 0 {
		return pipe
	}

	pipes = append([]gonzo.Pipe{pipe}, pipes...)

	ctx, cancel := context.WithCancel(pipe.Context())
	for _, pipe := range pipes {
		go func(c context.Context) {
			<-c.Done()
			cancel()
		}(pipe.Context())
	}

	out := make(chan gonzo.File)

	go func(out chan gonzo.File) {
		defer close(out)

		for _, p := range pipes {
			func(files <-chan gonzo.File) {
				for {
					select {
					case f, ok := <-files:
						if !ok {
							return
						}
						out <- f
					case <-ctx.Done():
						return
					}
				}
			}(p.Files())

			if ctx.Err() != nil {
				return
			}
		}
	}(out)

	return gonzo.NewPipe(ctx, out)
}
コード例 #3
0
ファイル: pipe.go プロジェクト: go-gonzo/util
// Merge concurrently Merges the output of multiple chan of gonzo.File into a pipe.
func Merge(ctx context.Context, pipes ...gonzo.Pipe) gonzo.Pipe {

	ctx, cancel := context.WithCancel(ctx)
	for _, pipe := range pipes {
		go func(c context.Context) {
			<-c.Done()
			cancel()
		}(pipe.Context())
	}

	out := make(chan gonzo.File)

	go func(out chan gonzo.File) {
		var wg sync.WaitGroup
		wg.Add(len(pipes))
		defer close(out)

		for _, p := range pipes {
			go func(p gonzo.Pipe) {
				defer wg.Done()
				files := p.Files()
				ctx := p.Context()
				for {
					select {
					case f, ok := <-files:
						if !ok {
							return
						}
						out <- f
					case <-ctx.Done():
						return
					}
				}
			}(p)
		}
		wg.Wait()
	}(out)

	return gonzo.NewPipe(ctx, out)
}
コード例 #4
0
ファイル: web.go プロジェクト: go-gonzo/web
// Gets  the list of urls and passes the results to output channel.
// It reports the progress to the Context using a ReadProgress proxy.
func Get(ctx context.Context, urls ...string) gonzo.Pipe {

	ctx, cancel := context.WithCancel(ctx)
	out := make(chan gonzo.File)
	client := &http.Client{}
	go func() {
		defer close(out)

		for _, url := range urls {

			if url == "" {
				ctx.Error("Empty URL.")
				cancel()
				return
			}
			select {
			case <-ctx.Done():
				ctx.Warn(context.Canceled)
				return
			default:
				ctx.Infof("Downloading %s", url)

				file, err := get(ctx, client, url)
				if err != nil {
					ctx.Error(err)
					cancel()
					break
				}

				//TODO: Add progress meter.
				//s, _ := file.Stat()
				//file.Reader = c.ReadProgress(file.Reader, "Downloading "+file.Path, s.Size())
				out <- file
			}
		}
	}()

	return gonzo.NewPipe(ctx, out)
}