Example #1
0
func (n Ned) Run() {
	os.MkdirAll(extractTo, 0777)
	os.MkdirAll(outputTo, 0777)

	files, err := ioutil.ReadDir(basePath)
	if err != nil {
		panic(err)
	}

	var wg sync.WaitGroup
	p := util.NewPool(3)
	for _, fileinfo := range files {
		if !strings.Contains(fileinfo.Name(), ".zip") {
			continue
		}
		wg.Add(1)
		go func(fileinfo os.FileInfo) {
			x := p.Borrow()
			if n.extractImage(fileinfo) {
				n.process(fileinfo)
				n.cleanup(fileinfo)
			}
			p.Return(x)
			wg.Done()
		}(fileinfo)
	}
	wg.Wait()
}
func (f FtpDownloader) Scan(threads int, matcher *regexp.Regexp) {
	// Scan for and download proper files
	// Note that this server doesn't like parallel downloads
	p := util.NewPool(threads)
	var wg sync.WaitGroup
	var i uint64
	entries, err := f.conn.List(".")
	if err != nil {
		panic(err)
	}
	fmt.Println("Downloading files")
	for _, entry := range entries {
		if matcher.MatchString(entry.Name) {
			wg.Add(1)
			go func(entry *ftp.Entry) {
				x := p.Borrow()
				f.Download(entry)
				p.Return(x)
				wg.Done()
			}(entry)
			i += entry.Size
		}
	}
	wg.Wait()
}