func TestPoolSerialLimitedWithoutBurst(t *testing.T) {
n := uint(1000)
op_per_sec := uint(10000)
worker := func(id uint, payload interface{}) interface{} {
return true
}
p := pool.NewRateLimitedPool(1, op_per_sec, 0, worker)
done := uint(0)
go func() {
for j := uint(0); j < n; j++ {
req := pool.NewJob(struct{}{})
p.Submit(req)
req.Result()
done++
}
}()
expected := float32(n) / float32(op_per_sec) * 1000
<-time.After(time.Duration(expected-expected/10) * time.Millisecond)
if done == n {
t.Fatalf("Pool completed work faster than specified rate")
}
}
func TestPoolSerialLimitedWithBurst(t *testing.T) {
n := uint(1000)
op_per_sec := uint(10000)
worker := func(id uint, payload interface{}) interface{} {
return true
}
p := pool.NewRateLimitedPool(1, op_per_sec, 1000, worker)
done := uint(0)
go func() {
time.Sleep(100 * time.Millisecond)
for j := uint(0); j < n; j++ {
req := pool.NewJob(struct{}{})
p.Submit(req)
req.Result()
done++
}
}()
// Processing should happen immediately because we have enough burst capacity
// to satisfy the jobs
<-time.After(110 * time.Millisecond)
if done != n {
t.Fatalf("Pool didn't use burst capacity")
}
}
func testPoolParallelLimitedWithoutBurst(t *testing.T, num_workers, num_req uint) {
runtime.GOMAXPROCS(4)
op_per_sec := uint(10000)
worker := func(id uint, payload interface{}) interface{} {
return true
}
p := pool.NewRateLimitedPool(num_workers, op_per_sec, 0, worker)
done := uint(0)
go func() {
for j := uint(0); j < num_req; j++ {
req := pool.NewJob(struct{}{})
p.Submit(req)
req.Result()
done++
}
}()
expected := float32(num_req) / float32(op_per_sec) * 1000
<-time.After(time.Duration(expected-expected/10) * time.Millisecond)
if done == num_req {
t.Fatalf("Pool completed work faster than specified rate")
}
runtime.GOMAXPROCS(runtime.GOMAXPROCS(-1))
}
return nil
}
type fetchTrips struct {
wg *sync.WaitGroup
page int
cb *citibike
}
var tripPool = pool.NewRateLimitedPool(10, 20, 10, func(id uint, payload interface{}) interface{} {
job := payload.(fetchTrips)
defer job.wg.Done()
doc, err := job.cb.get_trips_document(job.cb.trips_path + "?pageNumber=" + strconv.Itoa(job.page))
if err != nil {
return err
}
return job.cb.parse_trips(doc)
})
func (cb *citibike) get_all_trips(username string, cache TripCache) (Trips, error) {
doc, err := cb.get_trips_document(cb.trips_path)
if err != nil {
return nil, err
}
next_page, last_page := parse_navigation(doc, cb.trips_path)
var wg sync.WaitGroup