func TestPoolSerialLimitedWithoutBurst(t *testing.T) { n := uint(1000) op_per_sec := uint(10000) worker := func(id uint, payload interface{}) interface{} { return true } p := pool.NewRateLimitedPool(1, op_per_sec, 0, worker) done := uint(0) go func() { for j := uint(0); j < n; j++ { req := pool.NewJob(struct{}{}) p.Submit(req) req.Result() done++ } }() expected := float32(n) / float32(op_per_sec) * 1000 <-time.After(time.Duration(expected-expected/10) * time.Millisecond) if done == n { t.Fatalf("Pool completed work faster than specified rate") } }
func TestPoolSerialLimitedWithBurst(t *testing.T) { n := uint(1000) op_per_sec := uint(10000) worker := func(id uint, payload interface{}) interface{} { return true } p := pool.NewRateLimitedPool(1, op_per_sec, 1000, worker) done := uint(0) go func() { time.Sleep(100 * time.Millisecond) for j := uint(0); j < n; j++ { req := pool.NewJob(struct{}{}) p.Submit(req) req.Result() done++ } }() // Processing should happen immediately because we have enough burst capacity // to satisfy the jobs <-time.After(110 * time.Millisecond) if done != n { t.Fatalf("Pool didn't use burst capacity") } }
func testPoolParallelLimitedWithoutBurst(t *testing.T, num_workers, num_req uint) { runtime.GOMAXPROCS(4) op_per_sec := uint(10000) worker := func(id uint, payload interface{}) interface{} { return true } p := pool.NewRateLimitedPool(num_workers, op_per_sec, 0, worker) done := uint(0) go func() { for j := uint(0); j < num_req; j++ { req := pool.NewJob(struct{}{}) p.Submit(req) req.Result() done++ } }() expected := float32(num_req) / float32(op_per_sec) * 1000 <-time.After(time.Duration(expected-expected/10) * time.Millisecond) if done == num_req { t.Fatalf("Pool completed work faster than specified rate") } runtime.GOMAXPROCS(runtime.GOMAXPROCS(-1)) }
return nil } type fetchTrips struct { wg *sync.WaitGroup page int cb *citibike } var tripPool = pool.NewRateLimitedPool(10, 20, 10, func(id uint, payload interface{}) interface{} { job := payload.(fetchTrips) defer job.wg.Done() doc, err := job.cb.get_trips_document(job.cb.trips_path + "?pageNumber=" + strconv.Itoa(job.page)) if err != nil { return err } return job.cb.parse_trips(doc) }) func (cb *citibike) get_all_trips(username string, cache TripCache) (Trips, error) { doc, err := cb.get_trips_document(cb.trips_path) if err != nil { return nil, err } next_page, last_page := parse_navigation(doc, cb.trips_path) var wg sync.WaitGroup