func Watcher(c context.Context, task func(string), globs ...string) {
files, err := glob.Glob(globs...)
if err != nil {
c.Error(err)
return
}
// Create some channels
f := make(fileNameChan)
e := make(errorChan)
for matchpair := range files {
monitorFile(matchpair.Name, f, e)
}
go func() {
for {
select {
case fn := <-f:
task(fn)
case err := <-e:
c.Error(err)
}
}
}()
}
func Get(ctx context.Context, pkgs ...pkg.Package) gonzo.Pipe {
var all []gonzo.Pipe
for _, pkg := range pkgs {
if pkg.Dist.Tarball == "" {
ctx.Info("EMPTY", pkg.Name)
continue
}
all = append(all, get(ctx, pkg))
}
return util.Merge(ctx, all...)
}
func makestage(stage Stage, ctx context.Context, in <-chan File) Pipe {
out := make(chan File)
next, cancel := context.WithCancel(ctx)
go func() {
err := stage(ctx, in, out)
close(out)
if err != nil {
cancel()
ctx.Error(err)
}
}()
return pipe{files: out, context: next}
}
func Watcher(ctx context.Context, cb func(string), globs ...string) error {
files, err := glob.Glob(globs...)
if err != nil {
return err
}
w, err := fsnotify.NewWatcher()
if err != nil {
return err
}
for matchpair := range files {
w.Add(matchpair.Name)
}
throttled := throttle(50 * time.Millisecond)
go func() {
for {
select {
case event := <-w.Events:
//if event.Op&fsnotify.Write == fsnotify.Write {
//event.Op&fsnotify.Create == fsnotify.Create ||
throttled(func() {
cb(event.Name)
})
//}
case err := <-w.Errors:
if err != nil {
ctx.Error(err)
}
case <-ctx.Done():
w.Close()
return
}
}
}()
return nil
}
// Merge concurrently Merges the output of multiple chan of gonzo.File into a pipe.
func Merge(ctx context.Context, pipes ...gonzo.Pipe) gonzo.Pipe {
ctx, cancel := context.WithCancel(ctx)
for _, pipe := range pipes {
go func(c context.Context) {
<-c.Done()
cancel()
}(pipe.Context())
}
out := make(chan gonzo.File)
go func(out chan gonzo.File) {
var wg sync.WaitGroup
wg.Add(len(pipes))
defer close(out)
for _, p := range pipes {
go func(p gonzo.Pipe) {
defer wg.Done()
files := p.Files()
ctx := p.Context()
for {
select {
case f, ok := <-files:
if !ok {
return
}
out <- f
case <-ctx.Done():
return
}
}
}(p)
}
wg.Wait()
}(out)
return gonzo.NewPipe(ctx, out)
}
// Gets the list of urls and passes the results to output channel.
// It reports the progress to the Context using a ReadProgress proxy.
func Get(ctx context.Context, urls ...string) gonzo.Pipe {
ctx, cancel := context.WithCancel(ctx)
out := make(chan gonzo.File)
client := &http.Client{}
go func() {
defer close(out)
for _, url := range urls {
if url == "" {
ctx.Error("Empty URL.")
cancel()
return
}
select {
case <-ctx.Done():
ctx.Warn(context.Canceled)
return
default:
ctx.Infof("Downloading %s", url)
file, err := get(ctx, client, url)
if err != nil {
ctx.Error(err)
cancel()
break
}
//TODO: Add progress meter.
//s, _ := file.Stat()
//file.Reader = c.ReadProgress(file.Reader, "Downloading "+file.Path, s.Size())
out <- file
}
}
}()
return gonzo.NewPipe(ctx, out)
}
func get(ctx context.Context, release Release) gonzo.Pipe {
repo := fmt.Sprintf("%s/%s#%s", release.User, release.Repo, release.Tag)
ctx.Warn(repo)
return web.Get(
context.WithValue(ctx, "repo", repo),
fmt.Sprintf(
"https://codeload.github.com/%s/%s/tar.gz/%s",
release.User,
release.Repo,
release.Tag,
),
).Pipe(
gzip.Uncompress(),
tar.Untar(tar.Options{
StripComponenets: 1,
Pluck: release.Pluck,
}),
path.Rename(func(old string) string {
return filepath.Join(release.Repo, old)
}),
)
}
//Src returns a channel of gonzo.Files that match the provided patterns.
//TODO: ADD support for prefix to avoid all the util.Trims
func Src(ctx context.Context, globs ...string) gonzo.Pipe {
ctx, cancel := context.WithCancel(ctx)
files := make(chan gonzo.File)
pipe := gonzo.NewPipe(ctx, files)
//TODO: Parse globs here, check for invalid globs, split them into "filters".
go func() {
var err error
defer close(files)
fileslist, err := glob.Glob(globs...)
if err != nil {
ctx.Error(err)
return
}
for mp := range fileslist {
var (
file gonzo.File
base = glob.Dir(mp.Glob)
name = mp.Name
)
file, err = Read(mp.Name)
ctx = context.WithValue(ctx, "file", name)
if err == ErrIsDir {
ctx.Warn("fs.Src Ignored Directory.")
continue
}
if err != nil {
cancel()
ctx.Error(err)
return
}
file.FileInfo().SetBase(base)
file.FileInfo().SetName(name)
files <- file
}
}()
return pipe
}
// Concatenates all the files from the input channel
// and passes them to output channel with the given name.
func Concat(ctx context.Context, name string) gonzo.Stage {
return func(ctx context.Context, files <-chan gonzo.File, out chan<- gonzo.File) error {
var (
size int64
bigfile = new(bytes.Buffer)
)
err := func() error {
for {
select {
case f, ok := <-files:
if !ok {
return nil
}
ctx.Infof(
"Adding %s to %s",
filepath.Join(f.FileInfo().Base(), f.FileInfo().Name()),
name,
)
n, err := bigfile.ReadFrom(f)
if err != nil {
return err
}
bigfile.WriteRune('\n')
size += n + 1
f.Close()
case <-ctx.Done():
return ctx.Err()
}
}
}()
if err != nil {
return err
}
file := gonzo.NewFile(ioutil.NopCloser(bigfile), gonzo.NewFileInfo())
file.FileInfo().SetSize(size)
file.FileInfo().SetName(name)
out <- file
return nil
}
}