func TestRepeatSim(t *testing.T) {
clk := loadsim.NewSimClock()
req, err := http.NewRequest("GET", "", nil)
if err != nil {
log.Fatal(err)
}
agents := []loadsim.Agent{
&loadsim.RepeatAgent{
BaseRequest: req,
Clock: clk,
},
}
worker := makeSimWorker(clk)
stop := make(chan struct{})
resCh := loadsim.Simulate(agents, worker, clk, 10*time.Second)
go clk.Run(stop)
results := collectResults(resCh)[""]
close(stop)
if err := assertStatus(results, http.StatusOK); err != nil {
t.Error(err)
}
if err := assertWorkDurations(results, 100*time.Millisecond, time.Millisecond); err != nil {
t.Error(err)
}
count := len(results)
if count < 99 || count > 101 {
t.Errorf("expected 99-101 requests, got %d", count)
}
}
func TestPoolSim(t *testing.T) {
clk := loadsim.NewSimClock()
req, err := http.NewRequest("GET", "", nil)
if err != nil {
log.Fatal(err)
}
agents := []loadsim.Agent{
&loadsim.RepeatAgent{
BaseRequest: req,
Clock: clk,
ID: "repeat1",
},
&loadsim.RepeatAgent{
BaseRequest: req,
Clock: clk,
ID: "repeat2",
},
&loadsim.IntervalAgent{
BaseRequest: req,
Clock: clk,
Interval: 510 * time.Millisecond,
ID: "interval",
},
}
worker := &loadsim.WorkerPool{
Backlog: 10,
Timeout: 200 * time.Millisecond,
Workers: []loadsim.Worker{makeSimWorker(clk), makeSimWorker(clk)},
Clock: clk,
}
stop := make(chan struct{})
resCh := loadsim.Simulate(agents, worker, clk, 10*time.Second)
go clk.Run(stop)
agentResults := collectResults(resCh)
close(stop)
for id, results := range agentResults {
if err := assertStatus(results, http.StatusOK); err != nil {
t.Errorf("%s: %s", id, err)
}
if err := assertWorkDurations(results, 100*time.Millisecond, time.Millisecond); err != nil {
t.Errorf("%s: %s", id, err)
}
count := len(results)
switch id {
case "repeat1", "repeat2":
if count < 85 || count > 95 {
t.Errorf("%s: expected 85-95 requests, got %d", id, count)
}
case "interval":
if count != 20 {
t.Errorf("%s: expected 20 requests, got %d", id, count)
}
}
}
}
func listOverload() {
var keys []string
if envKeys := os.Getenv("STRIPE_KEYS"); envKeys != "" {
keys = strings.Split(envKeys, ",")
} else {
keys = []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P"}
}
start := time.Now()
workerCnt := 15
cfg := WorkerConfig{
Duration: 60 * time.Second,
Workers: workerCnt,
CPUs: 2,
Hosts: 2,
Backlog: 200,
Timeout: 40 * time.Second,
WallClockBurst: time.Duration(5*workerCnt) * time.Second,
WallClockRate: time.Duration(workerCnt) * time.Second,
ResourceMapper: func(req *http.Request) []loadsim.ResourceNeed {
if req.Method == "GET" {
return []loadsim.ResourceNeed{
{"time", 500},
{"CPU", 2000},
}
}
return []loadsim.ResourceNeed{
{"time", 1200},
{"CPU", 400},
}
},
}
prefix := "data"
simClock := loadsim.NewSimClock()
runners := []struct {
Name string
Run func(WorkerConfig, []loadsim.Agent, loadsim.Clock) []loadsim.Result
Clock loadsim.Clock
}{
{"sim", simRun, simClock},
//{"http", httpRun, &loadsim.WallClock{}},
}
listreq, err := http.NewRequest("GET", "https://qa-api.stripe.com/v1/charges?limit=100", nil)
if err != nil {
log.Fatal(err)
}
listreq.SetBasicAuth(keys[0], "")
for _, listAgents := range []int{1, 10, 60} {
for _, createInterval := range []time.Duration{0, 3 * time.Second} {
var rep string
if createInterval == 0 {
rep = "repeat"
} else {
rep = createInterval.String()
}
for _, runner := range runners {
var agents []loadsim.Agent
for i := 0; i < listAgents; i++ {
id := fmt.Sprintf("%s %s", listreq.Method, listreq.URL.Path)
delay := 20*time.Second + time.Millisecond*time.Duration(i*300)
agents = append(agents, &loadsim.DelayLimitAgent{
Agent: buildAgent(runner.Clock, id, listreq, nil, 0),
Delay: delay,
Limit: 60 * time.Second,
Clock: runner.Clock,
})
}
for i := 0; i < 6; i++ {
req, body := chargeCreate(keys[i+1])
id := fmt.Sprintf("%s %s", req.Method, req.URL.Path)
agents = append(agents, buildAgent(runner.Clock, id, req, body, createInterval))
}
results := runner.Run(cfg, agents, runner.Clock)
path := path.Join(prefix, fmt.Sprintf("%d_%s_%dlist@%s.tsv", start.Unix(), runner.Name, listAgents, rep))
if err := writeResults(results, path); err != nil {
log.Fatal(err)
}
}
}
}
}
func TestSimClockSimple(t *testing.T) {
stop := make(chan struct{})
clk := loadsim.NewSimClock()
go clk.Run(stop)
// Test a simple clock advancing
ticker1, tickStop1 := clk.Tick()
start, err := timeoutRead(ticker1)
if err != nil {
t.Fatal(err)
}
next, err := timeoutRead(ticker1)
if err != nil {
t.Fatal(err)
}
if next.Sub(start) != time.Millisecond {
t.Fatalf("clock advanced from %s by %s, not 1ms", start, next.Sub(start))
}
ticker2, tickStop2 := clk.Tick()
<-ticker1
// With the other clock started, we should read an equal number
// of ticks from both clocks but we can't be sure which we started
// from.
var now1, now2 time.Time
var count1, count2 int
for i := 0; i < 10; i++ {
select {
case next := <-ticker1:
if !(now1.IsZero() || next.Sub(now1) == time.Millisecond) {
t.Fatalf("clock1 advanced from %s to %s, not 1ms", now1, next)
}
now1 = next
count1++
case next := <-ticker2:
if !(now2.IsZero() || next.Sub(now2) == time.Millisecond) {
t.Fatalf("clock2 advanced from %s to %s, not 1ms", now1, next)
}
now2 = next
count2++
case <-time.After(time.Millisecond):
t.Fatalf("failed to read from ticker")
}
}
if count1 != count2 {
t.Fatalf("unequal clock advances, %d != %d", count1, count2)
}
if diff := now1.Sub(now2); diff > time.Millisecond || diff < -time.Millisecond {
t.Fatalf("clocks diverged, %s !~ %s", now1, now2)
}
// Ensure we're blocking on ticker1
select {
case <-ticker2:
default:
}
select {
case <-ticker2:
t.Fatalf("should be blocking on ticker1")
default:
}
// Stop ticker1 and ensure this unblocks us
close(tickStop1)
_, err = timeoutRead(ticker2)
if err != nil {
t.Fatalf("should have unblocked ticker2")
}
close(tickStop2)
}
func workerCounts() {
start := time.Now()
cpus := 4
hosts := 4
duration := 180 * time.Second
fuckeryDelay := duration / 6
fuckeryDuration := duration - fuckeryDelay*2
cfg := WorkerConfig{
Duration: duration,
CPUs: cpus,
Hosts: hosts,
Backlog: hosts * 100,
Timeout: 40 * time.Second,
}
prefix := "data"
clk := loadsim.NewSimClock()
normalMapper := func(req *http.Request) []loadsim.ResourceNeed {
if req.Method == "GET" {
total := 3000
wall := total / 5
return []loadsim.ResourceNeed{
{"time", wall},
{"CPU", total - wall},
}
}
total := 1350
cpu := total / 3
return []loadsim.ResourceNeed{
{"time", total - cpu},
{"CPU", cpu},
}
}
listreq, err := http.NewRequest("GET", "https://qa-api.stripe.com/v1/charges?limit=100", nil)
if err != nil {
log.Fatal(err)
}
listreq.SetBasicAuth("list", "")
var listAgents []loadsim.Agent
for i := 0; i < (hosts * cpus * 2); i++ {
listAgents = append(listAgents, buildAgent(clk, "list", listreq, nil, 0))
}
kerfuckery := []struct {
Name string
Agents []loadsim.Agent
ResourceMapper loadsim.ResourceMapper
}{
//{"none", nil, nil},
// 10% of charges see a 20 second delay in the response
/*{
"charge-net-delay",
nil,
func(req *http.Request) []loadsim.ResourceNeed {
needs := normalMapper(req)
if req.Method == "POST" && req.URL.Path == "/v1/charges" && rand.Float32() < 0.1 {
needs = append(needs, &loadsim.ResourceNeed{"time", 20000})
}
return needs
},
},*/
// 100% of charges see a 1.25 second delay in the response
/*{
"charge-scoring-delay",
nil,
func(req *http.Request) []loadsim.ResourceNeed {
needs := normalMapper(req)
if req.Method == "POST" && req.URL.Path == "/v1/charges" {
needs = append(needs, &loadsim.ResourceNeed{"time", 1250})
}
return needs
},
},*/
// A bunch of expensive list queries wreck havock
{"list", listAgents, nil},
}
for _, workerCnt := range []int{cpus + cpus/2, 2 * cpus, 4 * cpus, 8 * cpus} {
cfg.Workers = workerCnt
cfg.WallClockRate = time.Duration(workerCnt*hosts) / 2 * time.Second
cfg.WallClockBurst = cfg.WallClockRate * 5
for _, fuckery := range kerfuckery {
var agents []loadsim.Agent
for i := 0; i < hosts*cpus*3/2; i++ {
req, body := chargeCreate(strconv.Itoa(i))
agents = append(agents, buildAgent(clk, "charge", req, body, 3*time.Second))
}
for i, newAgent := range fuckery.Agents {
agents = append(agents, &loadsim.DelayLimitAgent{
Agent: newAgent,
Delay: fuckeryDelay + duration/1000*time.Duration(i),
Limit: duration - fuckeryDelay*2,
Clock: clk,
})
}
if fuckery.ResourceMapper != nil {
beginFuckery := clk.Now().Add(fuckeryDelay)
endFuckery := beginFuckery.Add(fuckeryDuration)
cfg.ResourceMapper = func(req *http.Request) []loadsim.ResourceNeed {
now := clk.Now()
if now.After(beginFuckery) && now.Before(endFuckery) {
return fuckery.ResourceMapper(req)
}
return normalMapper(req)
}
} else {
cfg.ResourceMapper = normalMapper
}
results := simRun(cfg, agents, clk)
path := path.Join(prefix, fmt.Sprintf("%d_%dworkers_%s.tsv", start.Unix(), workerCnt*hosts, fuckery.Name))
if err := writeResults(results, path); err != nil {
log.Fatal(err)
}
}
}
}
func qaListOverload() {
start := time.Now()
workerCnt := 15
hosts := 2
cpus := 2
cfg := WorkerConfig{
Duration: 210 * time.Second,
Workers: workerCnt,
CPUs: cpus,
Hosts: hosts,
Backlog: 100 * hosts,
Timeout: 40 * time.Second,
ResourceMapper: func(req *http.Request) []loadsim.ResourceNeed {
if req.Method == "GET" {
return []loadsim.ResourceNeed{
{"time", 500},
{"CPU", 2000},
}
}
return []loadsim.ResourceNeed{
{"time", 1200},
{"CPU", 400},
}
},
RequestOverhead: []loadsim.ResourceNeed{
{"time", 20},
{"CPU", 20},
},
}
prefix := "data"
clk := loadsim.NewSimClock()
listreq, err := http.NewRequest("GET", "https://qa-api.stripe.com/v1/charges?limit=100", nil)
if err != nil {
log.Fatal(err)
}
listreq.SetBasicAuth("list", "")
var agents []loadsim.Agent
for i := 0; i < 60; i++ {
id := fmt.Sprintf("%s %s", listreq.Method, listreq.URL.Path)
delay := 30*time.Second + time.Millisecond*time.Duration(i*200)
agents = append(agents, &loadsim.DelayLimitAgent{
Agent: buildAgent(clk, id, listreq, nil, 0),
Delay: delay,
Limit: 120 * time.Second,
Clock: clk,
})
}
for i := 0; i < 6; i++ {
id := fmt.Sprintf("charge%d", i)
req, body := chargeCreate(id)
agents = append(agents, buildAgent(clk, id, req, body, 0))
}
cases := []struct {
Name string
WallClockRate time.Duration
}{
{"no-limiter", 0},
{"limiter", time.Duration(workerCnt*hosts/2) * time.Second},
}
for _, simcase := range cases {
cfg.WallClockRate = simcase.WallClockRate
cfg.WallClockBurst = 5 * cfg.WallClockRate
results := simRun(cfg, agents, clk)
path := path.Join(prefix, fmt.Sprintf("%d_%s_qalist.tsv", start.Unix(), simcase.Name))
if err := writeResults(results, path); err != nil {
log.Fatal(err)
}
}
}