Example #1
0
// Concatenates all the files from the input channel
// and passes them to output channel with the given name.
func Concat(ctx context.Context, name string) gonzo.Stage {
	return func(ctx context.Context, files <-chan gonzo.File, out chan<- gonzo.File) error {

		var (
			size    int64
			bigfile = new(bytes.Buffer)
		)

		err := func() error {
			for {
				select {
				case f, ok := <-files:
					if !ok {
						return nil
					}

					ctx.Infof(
						"Adding %s to %s",
						filepath.Join(f.FileInfo().Base(), f.FileInfo().Name()),
						name,
					)

					n, err := bigfile.ReadFrom(f)
					if err != nil {
						return err
					}
					bigfile.WriteRune('\n')
					size += n + 1

					f.Close()
				case <-ctx.Done():
					return ctx.Err()
				}
			}
		}()

		if err != nil {
			return err
		}

		file := gonzo.NewFile(ioutil.NopCloser(bigfile), gonzo.NewFileInfo())
		file.FileInfo().SetSize(size)
		file.FileInfo().SetName(name)
		out <- file
		return nil
	}
}
Example #2
0
func Watcher(ctx context.Context, cb func(string), globs ...string) error {

	files, err := glob.Glob(globs...)

	if err != nil {
		return err
	}

	w, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	for matchpair := range files {
		w.Add(matchpair.Name)
	}

	throttled := throttle(50 * time.Millisecond)
	go func() {
		for {
			select {
			case event := <-w.Events:
				//if event.Op&fsnotify.Write == fsnotify.Write {
				//event.Op&fsnotify.Create == fsnotify.Create ||
				throttled(func() {
					cb(event.Name)
				})
				//}
			case err := <-w.Errors:
				if err != nil {
					ctx.Error(err)
				}
			case <-ctx.Done():
				w.Close()
				return
			}
		}
	}()

	return nil
}
Example #3
0
// Merge concurrently Merges the output of multiple chan of gonzo.File into a pipe.
func Merge(ctx context.Context, pipes ...gonzo.Pipe) gonzo.Pipe {

	ctx, cancel := context.WithCancel(ctx)
	for _, pipe := range pipes {
		go func(c context.Context) {
			<-c.Done()
			cancel()
		}(pipe.Context())
	}

	out := make(chan gonzo.File)

	go func(out chan gonzo.File) {
		var wg sync.WaitGroup
		wg.Add(len(pipes))
		defer close(out)

		for _, p := range pipes {
			go func(p gonzo.Pipe) {
				defer wg.Done()
				files := p.Files()
				ctx := p.Context()
				for {
					select {
					case f, ok := <-files:
						if !ok {
							return
						}
						out <- f
					case <-ctx.Done():
						return
					}
				}
			}(p)
		}
		wg.Wait()
	}(out)

	return gonzo.NewPipe(ctx, out)
}
Example #4
0
File: web.go Project: go-gonzo/web
// Gets  the list of urls and passes the results to output channel.
// It reports the progress to the Context using a ReadProgress proxy.
func Get(ctx context.Context, urls ...string) gonzo.Pipe {

	ctx, cancel := context.WithCancel(ctx)
	out := make(chan gonzo.File)
	client := &http.Client{}
	go func() {
		defer close(out)

		for _, url := range urls {

			if url == "" {
				ctx.Error("Empty URL.")
				cancel()
				return
			}
			select {
			case <-ctx.Done():
				ctx.Warn(context.Canceled)
				return
			default:
				ctx.Infof("Downloading %s", url)

				file, err := get(ctx, client, url)
				if err != nil {
					ctx.Error(err)
					cancel()
					break
				}

				//TODO: Add progress meter.
				//s, _ := file.Stat()
				//file.Reader = c.ReadProgress(file.Reader, "Downloading "+file.Path, s.Size())
				out <- file
			}
		}
	}()

	return gonzo.NewPipe(ctx, out)
}