func (n Ned) Run() {
os.MkdirAll(extractTo, 0777)
os.MkdirAll(outputTo, 0777)
files, err := ioutil.ReadDir(basePath)
if err != nil {
panic(err)
}
var wg sync.WaitGroup
p := util.NewPool(3)
for _, fileinfo := range files {
if !strings.Contains(fileinfo.Name(), ".zip") {
continue
}
wg.Add(1)
go func(fileinfo os.FileInfo) {
x := p.Borrow()
if n.extractImage(fileinfo) {
n.process(fileinfo)
n.cleanup(fileinfo)
}
p.Return(x)
wg.Done()
}(fileinfo)
}
wg.Wait()
}
func (f FtpDownloader) Scan(threads int, matcher *regexp.Regexp) {
// Scan for and download proper files
// Note that this server doesn't like parallel downloads
p := util.NewPool(threads)
var wg sync.WaitGroup
var i uint64
entries, err := f.conn.List(".")
if err != nil {
panic(err)
}
fmt.Println("Downloading files")
for _, entry := range entries {
if matcher.MatchString(entry.Name) {
wg.Add(1)
go func(entry *ftp.Entry) {
x := p.Borrow()
f.Download(entry)
p.Return(x)
wg.Done()
}(entry)
i += entry.Size
}
}
wg.Wait()
}