func TestFragmentationSlowReader(t *testing.T) {
startReading, handlerComplete := make(chan struct{}), make(chan struct{})
handler := func(ctx context.Context, call *InboundCall) {
<-ctx.Done()
<-startReading
_, err := raw.ReadArgs(call)
assert.Error(t, err, "ReadArgs should fail since frames will be dropped due to slow reading")
close(handlerComplete)
}
// Inbound forward will timeout and cause a warning log.
opts := testutils.NewOpts().AddLogFilter("Unable to forward frame", 1)
WithVerifiedServer(t, opts, func(ch *Channel, hostPort string) {
ch.Register(HandlerFunc(handler), "echo")
arg2 := testutils.RandBytes(MaxFramePayloadSize * MexChannelBufferSize)
arg3 := testutils.RandBytes(MaxFramePayloadSize * (MexChannelBufferSize + 1))
ctx, cancel := NewContext(testutils.Timeout(15 * time.Millisecond))
defer cancel()
_, _, _, err := raw.Call(ctx, ch, hostPort, testServiceName, "echo", arg2, arg3)
assert.Error(t, err, "Call should timeout due to slow reader")
close(startReading)
<-handlerComplete
})
goroutines.VerifyNoLeaks(t, nil)
}
func TestWriteErrorAfterTimeout(t *testing.T) {
// TODO: Make this test block at different points (e.g. before, during read/write).
WithVerifiedServer(t, nil, func(ch *Channel, hostPort string) {
timedOut := make(chan struct{})
done := make(chan struct{})
handler := func(ctx context.Context, call *InboundCall) {
<-ctx.Done()
<-timedOut
_, err := raw.ReadArgs(call)
assert.Equal(t, ErrTimeout, err, "Read args should fail with timeout")
response := call.Response()
assert.Equal(t, ErrTimeout, response.SendSystemError(ErrServerBusy), "SendSystemError should fail")
close(done)
}
ch.Register(HandlerFunc(handler), "call")
ctx, cancel := NewContext(testutils.Timeout(20 * time.Millisecond))
defer cancel()
_, _, _, err := raw.Call(ctx, ch, hostPort, testServiceName, "call", nil, testutils.RandBytes(100000))
assert.Equal(t, err, ErrTimeout, "Call should timeout")
close(timedOut)
<-done
})
goroutines.VerifyNoLeaks(t, nil)
}
func TestWriteErrorAfterTimeout(t *testing.T) {
// TODO: Make this test block at different points (e.g. before, during read/write).
testutils.WithTestServer(t, nil, func(ts *testutils.TestServer) {
timedOut := make(chan struct{})
done := make(chan struct{})
handler := func(ctx context.Context, call *InboundCall) {
<-ctx.Done()
<-timedOut
_, err := raw.ReadArgs(call)
assert.Equal(t, ErrTimeout, err, "Read args should fail with timeout")
response := call.Response()
assert.Equal(t, ErrTimeout, response.SendSystemError(ErrServerBusy), "SendSystemError should fail")
close(done)
}
ts.Register(HandlerFunc(handler), "call")
ctx, cancel := NewContext(testutils.Timeout(30 * time.Millisecond))
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "call", nil, testutils.RandBytes(100000))
assert.Equal(t, err, ErrTimeout, "Call should timeout")
close(timedOut)
select {
case <-done:
case <-time.After(time.Second):
t.Errorf("Handler not called, timeout may be too low")
}
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "call").Failed("timeout").End()
ts.AssertRelayStats(calls)
})
}
func TestFragmentationSlowReader(t *testing.T) {
startReading, handlerComplete := make(chan struct{}), make(chan struct{})
handler := func(ctx context.Context, call *InboundCall) {
<-startReading
_, err := raw.ReadArgs(call)
assert.Error(t, err, "ReadArgs should fail since frames will be dropped due to slow reading")
close(handlerComplete)
}
WithVerifiedServer(t, nil, func(ch *Channel, hostPort string) {
ch.Register(HandlerFunc(handler), "echo")
arg2 := testutils.RandBytes(MaxFramePayloadSize * MexChannelBufferSize)
arg3 := testutils.RandBytes(MaxFramePayloadSize * (MexChannelBufferSize + 1))
ctx, cancel := NewContext(10 * time.Millisecond)
defer cancel()
_, _, _, err := raw.Call(ctx, ch, hostPort, testServiceName, "echo", arg2, arg3)
assert.Error(t, err, "Call should timeout due to slow reader")
close(startReading)
<-handlerComplete
})
VerifyNoBlockedGoroutines(t)
}
func TestServerBusy(t *testing.T) {
WithVerifiedServer(t, nil, func(ch *Channel, hostPort string) {
ch.Register(ErrorHandlerFunc(func(ctx context.Context, call *InboundCall) error {
if _, err := raw.ReadArgs(call); err != nil {
return err
}
return ErrServerBusy
}), "busy")
ctx, cancel := NewContext(time.Second)
defer cancel()
_, _, _, err := raw.Call(ctx, ch, hostPort, testServiceName, "busy", []byte("Arg2"), []byte("Arg3"))
require.NotNil(t, err)
assert.Equal(t, ErrCodeBusy, GetSystemErrorCode(err), "err: %v", err)
})
}
func TestWriteArg3AfterTimeout(t *testing.T) {
// The channel reads and writes during timeouts, causing warning logs.
opts := testutils.NewOpts().DisableLogVerification()
testutils.WithTestServer(t, opts, func(ts *testutils.TestServer) {
timedOut := make(chan struct{})
handler := func(ctx context.Context, call *InboundCall) {
_, err := raw.ReadArgs(call)
assert.NoError(t, err, "Read args failed")
response := call.Response()
assert.NoError(t, NewArgWriter(response.Arg2Writer()).Write(nil), "Write Arg2 failed")
writer, err := response.Arg3Writer()
assert.NoError(t, err, "Arg3Writer failed")
for {
if _, err := writer.Write(testutils.RandBytes(4096)); err != nil {
assert.Equal(t, err, ErrTimeout, "Handler should timeout")
close(timedOut)
return
}
runtime.Gosched()
}
}
ts.Register(HandlerFunc(handler), "call")
ctx, cancel := NewContext(testutils.Timeout(50 * time.Millisecond))
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "call", nil, nil)
assert.Equal(t, err, ErrTimeout, "Call should timeout")
// Wait for the write to complete, make sure there's no errors.
select {
case <-time.After(testutils.Timeout(30 * time.Millisecond)):
t.Errorf("Handler should have failed due to timeout")
case <-timedOut:
}
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "call").Failed("timeout").Succeeded().End()
ts.AssertRelayStats(calls)
})
}
func TestUnexpectedHandlerError(t *testing.T) {
opts := testutils.NewOpts().
AddLogFilter("Unexpected handler error", 1)
WithVerifiedServer(t, opts, func(ch *Channel, hostPort string) {
ch.Register(ErrorHandlerFunc(func(ctx context.Context, call *InboundCall) error {
if _, err := raw.ReadArgs(call); err != nil {
return err
}
return fmt.Errorf("nope")
}), "nope")
ctx, cancel := NewContext(time.Second)
defer cancel()
_, _, _, err := raw.Call(ctx, ch, hostPort, testServiceName, "nope", []byte("Arg2"), []byte("Arg3"))
require.NotNil(t, err)
assert.Equal(t, ErrCodeUnexpected, GetSystemErrorCode(err), "err: %v", err)
})
}
func TestServerBusy(t *testing.T) {
testutils.WithTestServer(t, nil, func(ts *testutils.TestServer) {
ts.Register(ErrorHandlerFunc(func(ctx context.Context, call *InboundCall) error {
if _, err := raw.ReadArgs(call); err != nil {
return err
}
return ErrServerBusy
}), "busy")
ctx, cancel := NewContext(time.Second)
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "busy", []byte("Arg2"), []byte("Arg3"))
require.NotNil(t, err)
assert.Equal(t, ErrCodeBusy, GetSystemErrorCode(err), "err: %v", err)
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "busy").Failed("busy").End()
ts.AssertRelayStats(calls)
})
}
func TestFragmentationSlowReader(t *testing.T) {
// Inbound forward will timeout and cause a warning log.
opts := testutils.NewOpts().
AddLogFilter("Unable to forward frame", 1).
AddLogFilter("Connection error", 1)
testutils.WithTestServer(t, opts, func(ts *testutils.TestServer) {
startReading, handlerComplete := make(chan struct{}), make(chan struct{})
handler := func(ctx context.Context, call *InboundCall) {
<-startReading
<-ctx.Done()
_, err := raw.ReadArgs(call)
assert.Error(t, err, "ReadArgs should fail since frames will be dropped due to slow reading")
close(handlerComplete)
}
ts.Register(HandlerFunc(handler), "echo")
arg2 := testutils.RandBytes(MaxFramePayloadSize * MexChannelBufferSize)
arg3 := testutils.RandBytes(MaxFramePayloadSize * (MexChannelBufferSize + 1))
ctx, cancel := NewContext(testutils.Timeout(30 * time.Millisecond))
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "echo", arg2, arg3)
assert.Error(t, err, "Call should timeout due to slow reader")
close(startReading)
select {
case <-handlerComplete:
case <-time.After(testutils.Timeout(70 * time.Millisecond)):
t.Errorf("Handler not called, context timeout may be too low")
}
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "echo").Failed("timeout").End()
ts.AssertRelayStats(calls)
})
}
func TestWriteArg3AfterTimeout(t *testing.T) {
// The channel reads and writes during timeouts, causing warning logs.
opts := testutils.NewOpts().DisableLogVerification()
WithVerifiedServer(t, opts, func(ch *Channel, hostPort string) {
timedOut := make(chan struct{})
handler := func(ctx context.Context, call *InboundCall) {
_, err := raw.ReadArgs(call)
assert.NoError(t, err, "Read args failed")
response := call.Response()
assert.NoError(t, NewArgWriter(response.Arg2Writer()).Write(nil), "Write Arg2 failed")
writer, err := response.Arg3Writer()
assert.NoError(t, err, "Arg3Writer failed")
for {
if _, err := writer.Write(testutils.RandBytes(4096)); err != nil {
assert.Equal(t, err, ErrTimeout, "Handler should timeout")
close(timedOut)
return
}
runtime.Gosched()
}
}
ch.Register(HandlerFunc(handler), "call")
ctx, cancel := NewContext(20 * time.Millisecond)
defer cancel()
_, _, _, err := raw.Call(ctx, ch, hostPort, testServiceName, "call", nil, nil)
assert.Equal(t, err, ErrTimeout, "Call should timeout")
// Wait for the write to complete, make sure there's no errors.
select {
case <-time.After(30 * time.Millisecond):
t.Errorf("Handler should have failed due to timeout")
case <-timedOut:
}
})
VerifyNoBlockedGoroutines(t)
}
func TestUnexpectedHandlerError(t *testing.T) {
opts := testutils.NewOpts().
AddLogFilter("Unexpected handler error", 1)
testutils.WithTestServer(t, opts, func(ts *testutils.TestServer) {
ts.Register(ErrorHandlerFunc(func(ctx context.Context, call *InboundCall) error {
if _, err := raw.ReadArgs(call); err != nil {
return err
}
return fmt.Errorf("nope")
}), "nope")
ctx, cancel := NewContext(time.Second)
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "nope", []byte("Arg2"), []byte("Arg3"))
require.NotNil(t, err)
assert.Equal(t, ErrCodeUnexpected, GetSystemErrorCode(err), "err: %v", err)
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "nope").Failed("unexpected-error").End()
ts.AssertRelayStats(calls)
})
}