func serverExample() (err error) { bucket := ratelimit.NewFromRate(10*1024, 10*1024, 0) content := strings.Repeat(".", 100*1024) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { // all server threads share 10k bandwidth by using same bucket wrappedWriter := ratelimit.NewWriter(w, bucket) fmt.Fprint(wrappedWriter, content) }) http.HandleFunc("/10k", func(w http.ResponseWriter, r *http.Request) { // each thread has 10k bandwidth bucket := ratelimit.NewFromRate(10*1024, 10*1024, 0) wrappedWriter := ratelimit.NewWriter(w, bucket) fmt.Fprint(wrappedWriter, content) }) return http.ListenAndServe(":8000", nil) }
func clientExample() error { resp, err := http.Get("http://127.0.0.1:8000/") if err != nil { return err } defer resp.Body.Close() bucket := ratelimit.NewFromRate(10*1024, 10*1024, 0) wrappedReader := ratelimit.NewReader(resp.Body, bucket) if _, err := ioutil.ReadAll(wrappedReader); err != nil { return err } return nil }
func main() { log.Printf("Reading config file from %s", cfgFile) cfgStr, err := ioutil.ReadFile(cfgFile) if err != nil { log.Fatalf("Cannot read config from %s: %s", cfgFile, err) } cfg, err := ParseConfig(string(cfgStr)) if err != nil { log.Fatalf("Error parsing config files: %s", err) } nthreads := cfg.GetInt("nthreads") if nthreads < 1 { nthreads = 1 } var bucket *ratelimit.Bucket = nil rate := cfg.GetInt("ratelimit") if rate > 0 { log.Printf("Limit overall transfer rate to %dkb/s", rate) bucket = ratelimit.NewFromRate(float64(rate*1024), int64(rate*1024), 0) } dlMgr := NewManager( func(u *url.URL) string { return fmt.Sprintf("URL scheme %s of %s is not supported", u.Scheme, u) }, bucket, http.DefaultClient, nthreads, ) log.Printf("Path holding temp files(skel_path): %s", cfg.Variables["skel_path"]) log.Printf("Path holding mirrored files(mirror_path): %s", cfg.Variables["mirror_path"]) log.Printf("Default architecture: %s", cfg.Variables["defaultarch"]) log.Printf("Spawning %d goroutines to download packages.", nthreads) ch := make(chan Package) finish := make(chan int) for i := 0; i < nthreads; i++ { go worker(i, cfg, dlMgr, ch, finish) } // download info files, process packages file and generate file list debs := make(map[string]bool) for _, repo := range cfg.Repositories { repo.DownloadInfoFiles(cfg, dlMgr) for _, comp := range repo.Components { pkgFile := cfg.SkelPath(repo.Packages(comp)) var f io.ReadCloser f, err := os.Open(pkgFile) if err != nil { // maybe file not found, use gzipped file instead pkgFile = cfg.SkelPath(repo.PackagesGZ(comp)) f, err = os.Open(pkgFile) if err == nil { f, err = gzip.NewReader(f) } } if err != nil { log.Fatalf("Cannot open package file %s: %s", pkgFile, err) } pkgs, err := ParsePackage(repo, f) if err != nil { log.Fatalf("Cannot parse Packages file %s: %s", pkgFile, err) } f.Close() for _, p := range pkgs { m := cfg.MirrorPath(p.URL) debs[m] = true ch <- p } } } close(ch) log.Printf("Got %d package files ... ", len(debs)) // wait for download finish for i := 0; i < nthreads; i++ { <-finish } for c := range cfg.Clean { log.Printf("Cleaning %s", c) clean(c, cfg, debs) } if !dryRun { movefiles(cfg.Variables["skel_path"], cfg.Variables["mirror_path"]) } }