Exemple #1
0
func TestPoolSerialLimitedWithoutBurst(t *testing.T) {
	n := uint(1000)
	op_per_sec := uint(10000)

	worker := func(id uint, payload interface{}) interface{} {
		return true
	}
	p := pool.NewRateLimitedPool(1, op_per_sec, 0, worker)

	done := uint(0)
	go func() {
		for j := uint(0); j < n; j++ {
			req := pool.NewJob(struct{}{})
			p.Submit(req)
			req.Result()
			done++
		}
	}()

	expected := float32(n) / float32(op_per_sec) * 1000
	<-time.After(time.Duration(expected-expected/10) * time.Millisecond)
	if done == n {
		t.Fatalf("Pool completed work faster than specified rate")
	}
}
Exemple #2
0
func TestPoolSerialLimitedWithBurst(t *testing.T) {
	n := uint(1000)
	op_per_sec := uint(10000)

	worker := func(id uint, payload interface{}) interface{} {
		return true
	}
	p := pool.NewRateLimitedPool(1, op_per_sec, 1000, worker)

	done := uint(0)
	go func() {
		time.Sleep(100 * time.Millisecond)
		for j := uint(0); j < n; j++ {
			req := pool.NewJob(struct{}{})
			p.Submit(req)
			req.Result()
			done++
		}
	}()

	// Processing should happen immediately because we have enough burst capacity
	// to satisfy the jobs
	<-time.After(110 * time.Millisecond)
	if done != n {
		t.Fatalf("Pool didn't use burst capacity")
	}
}
Exemple #3
0
func testPoolParallelLimitedWithoutBurst(t *testing.T, num_workers, num_req uint) {
	runtime.GOMAXPROCS(4)

	op_per_sec := uint(10000)

	worker := func(id uint, payload interface{}) interface{} {
		return true
	}
	p := pool.NewRateLimitedPool(num_workers, op_per_sec, 0, worker)

	done := uint(0)
	go func() {
		for j := uint(0); j < num_req; j++ {
			req := pool.NewJob(struct{}{})
			p.Submit(req)
			req.Result()
			done++
		}
	}()

	expected := float32(num_req) / float32(op_per_sec) * 1000
	<-time.After(time.Duration(expected-expected/10) * time.Millisecond)
	if done == num_req {
		t.Fatalf("Pool completed work faster than specified rate")
	}

	runtime.GOMAXPROCS(runtime.GOMAXPROCS(-1))
}
Exemple #4
0
	return nil
}

type fetchTrips struct {
	wg   *sync.WaitGroup
	page int
	cb   *citibike
}

var tripPool = pool.NewRateLimitedPool(10, 20, 10, func(id uint, payload interface{}) interface{} {
	job := payload.(fetchTrips)
	defer job.wg.Done()

	doc, err := job.cb.get_trips_document(job.cb.trips_path + "?pageNumber=" + strconv.Itoa(job.page))
	if err != nil {
		return err
	}

	return job.cb.parse_trips(doc)
})

func (cb *citibike) get_all_trips(username string, cache TripCache) (Trips, error) {
	doc, err := cb.get_trips_document(cb.trips_path)
	if err != nil {
		return nil, err
	}

	next_page, last_page := parse_navigation(doc, cb.trips_path)

	var wg sync.WaitGroup