// Fetch http file url to destination dest, with or without progress.
func FetchHTTPFile(url string, dest string, progress bool) (err error) {
gologit.Debugf("Creating file: %s\n", dest)
out, err := os.Create(dest)
if err != nil {
return err
}
defer out.Close()
var r io.Reader
gologit.Debugf("Fetching url: %s\n", url)
resp, err := http.Get(url)
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("Server return non-200 status: %v", resp.Status)
}
msgPrefix := fmt.Sprintf("%s: ", path.Base(dest))
var bar *pb.ProgressBar
i, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
if i > 0 && progress {
bar = pb.New(i).Prefix(msgPrefix).SetUnits(pb.U_BYTES)
bar.ShowSpeed = true
bar.RefreshRate = time.Millisecond * 700
bar.ShowFinalTime = false
bar.ShowTimeLeft = false
bar.Start()
defer bar.Finish()
r = bar.NewProxyReader(resp.Body)
} else {
r = resp.Body
}
_, err = io.Copy(out, r)
return err
}
func main() {
flag.Usage = usage
help := flag.Bool("help", false, "show this message")
version := flag.Bool("version", false, "show version")
failpath := flag.String("faildir", "", "dir where failed torrentzips should be copied")
flag.Parse()
if *help {
flag.Usage()
os.Exit(0)
}
if *version {
fmt.Fprintf(os.Stdout, "%s version %s, Copyright (c) 2013 Uwe Hoffmann. All rights reserved.\n", os.Args[0], versionStr)
os.Exit(0)
}
if *failpath == "" {
flag.Usage()
os.Exit(0)
}
cv := new(countVisitor)
for _, name := range flag.Args() {
fmt.Fprintf(os.Stdout, "initial scan of %s to determine amount of work\n", name)
err := filepath.Walk(name, cv.visit)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to count in dir %s: %v\n", name, err)
os.Exit(1)
}
}
mg := int(cv.numBytes / megabyte)
fmt.Fprintf(os.Stdout, "found %d files and %d MB to do. starting work...\n", cv.numFiles, mg)
var byteProgress *pb.ProgressBar
if mg > 10 {
pb.BarStart = "MB ["
byteProgress = pb.New(mg)
byteProgress.RefreshRate = 5 * time.Second
byteProgress.ShowCounters = true
byteProgress.Start()
}
inwork := make(chan *workUnit)
sv := &scanVisitor{
inwork: inwork,
}
wg := new(sync.WaitGroup)
wg.Add(cv.numFiles)
for i := 0; i < 8; i++ {
worker := &testWorker{
byteProgress: byteProgress,
failpath: *failpath,
inwork: inwork,
wg: wg,
}
go worker.run()
}
for _, name := range flag.Args() {
err := filepath.Walk(name, sv.visit)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to scan dir %s: %v\n", name, err)
os.Exit(1)
}
}
wg.Wait()
close(inwork)
if byteProgress != nil {
byteProgress.Set(int(byteProgress.Total))
byteProgress.Finish()
}
fmt.Fprintf(os.Stdout, "Done.\n")
}