Beispiel #1
0
func serverExample() (err error) {
	bucket := ratelimit.NewFromRate(10*1024, 10*1024, 0)
	content := strings.Repeat(".", 100*1024)
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		// all server threads share 10k bandwidth by using same bucket
		wrappedWriter := ratelimit.NewWriter(w, bucket)
		fmt.Fprint(wrappedWriter, content)
	})
	http.HandleFunc("/10k", func(w http.ResponseWriter, r *http.Request) {
		// each thread has 10k bandwidth
		bucket := ratelimit.NewFromRate(10*1024, 10*1024, 0)
		wrappedWriter := ratelimit.NewWriter(w, bucket)
		fmt.Fprint(wrappedWriter, content)
	})

	return http.ListenAndServe(":8000", nil)
}
Beispiel #2
0
func clientExample() error {
	resp, err := http.Get("http://127.0.0.1:8000/")
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	bucket := ratelimit.NewFromRate(10*1024, 10*1024, 0)
	wrappedReader := ratelimit.NewReader(resp.Body, bucket)
	if _, err := ioutil.ReadAll(wrappedReader); err != nil {
		return err
	}
	return nil
}
func main() {
	log.Printf("Reading config file from %s", cfgFile)
	cfgStr, err := ioutil.ReadFile(cfgFile)
	if err != nil {
		log.Fatalf("Cannot read config from %s: %s", cfgFile, err)
	}

	cfg, err := ParseConfig(string(cfgStr))
	if err != nil {
		log.Fatalf("Error parsing config files: %s", err)
	}

	nthreads := cfg.GetInt("nthreads")
	if nthreads < 1 {
		nthreads = 1
	}

	var bucket *ratelimit.Bucket = nil
	rate := cfg.GetInt("ratelimit")
	if rate > 0 {
		log.Printf("Limit overall transfer rate to %dkb/s", rate)
		bucket = ratelimit.NewFromRate(float64(rate*1024), int64(rate*1024), 0)
	}

	dlMgr := NewManager(
		func(u *url.URL) string {
			return fmt.Sprintf("URL scheme %s of %s is not supported", u.Scheme, u)
		},
		bucket,
		http.DefaultClient,
		nthreads,
	)

	log.Printf("Path holding temp files(skel_path): %s", cfg.Variables["skel_path"])
	log.Printf("Path holding mirrored files(mirror_path): %s", cfg.Variables["mirror_path"])
	log.Printf("Default architecture: %s", cfg.Variables["defaultarch"])
	log.Printf("Spawning %d goroutines to download packages.", nthreads)

	ch := make(chan Package)
	finish := make(chan int)

	for i := 0; i < nthreads; i++ {
		go worker(i, cfg, dlMgr, ch, finish)
	}

	// download info files, process packages file and generate file list
	debs := make(map[string]bool)
	for _, repo := range cfg.Repositories {
		repo.DownloadInfoFiles(cfg, dlMgr)

		for _, comp := range repo.Components {
			pkgFile := cfg.SkelPath(repo.Packages(comp))
			var f io.ReadCloser
			f, err := os.Open(pkgFile)
			if err != nil {
				// maybe file not found, use gzipped file instead
				pkgFile = cfg.SkelPath(repo.PackagesGZ(comp))
				f, err = os.Open(pkgFile)
				if err == nil {
					f, err = gzip.NewReader(f)
				}
			}
			if err != nil {
				log.Fatalf("Cannot open package file %s: %s", pkgFile, err)
			}
			pkgs, err := ParsePackage(repo, f)
			if err != nil {
				log.Fatalf("Cannot parse Packages file %s: %s", pkgFile, err)
			}
			f.Close()

			for _, p := range pkgs {
				m := cfg.MirrorPath(p.URL)
				debs[m] = true
				ch <- p
			}
		}
	}
	close(ch)
	log.Printf("Got %d package files ... ", len(debs))

	// wait for download finish
	for i := 0; i < nthreads; i++ {
		<-finish
	}

	for c := range cfg.Clean {
		log.Printf("Cleaning %s", c)
		clean(c, cfg, debs)
	}

	if !dryRun {
		movefiles(cfg.Variables["skel_path"], cfg.Variables["mirror_path"])
	}

}