func TestFragmentation(t *testing.T) {
testutils.WithTestServer(t, nil, func(ts *testutils.TestServer) {
ts.Register(raw.Wrap(newTestHandler(t)), "echo")
arg2 := make([]byte, MaxFramePayloadSize*2)
for i := 0; i < len(arg2); i++ {
arg2[i] = byte('a' + (i % 10))
}
arg3 := make([]byte, MaxFramePayloadSize*3)
for i := 0; i < len(arg3); i++ {
arg3[i] = byte('A' + (i % 10))
}
ctx, cancel := NewContext(time.Second)
defer cancel()
respArg2, respArg3, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "echo", arg2, arg3)
require.NoError(t, err)
assert.Equal(t, arg2, respArg2)
assert.Equal(t, arg3, respArg3)
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "echo").Succeeded().End()
ts.AssertRelayStats(calls)
})
}
func TestReadTimeout(t *testing.T) {
// The error frame may fail to send since the connection closes before the handler sends it
// or the handler connection may be closed as it sends when the other side closes the conn.
opts := testutils.NewOpts().
AddLogFilter("Couldn't send outbound error frame", 1).
AddLogFilter("Connection error", 1, "site", "read frames").
AddLogFilter("Connection error", 1, "site", "write frames").
AddLogFilter("simpleHandler OnError", 1,
"error", "failed to send error frame, connection state connectionClosed")
testutils.WithTestServer(t, opts, func(ts *testutils.TestServer) {
sn := ts.ServiceName()
calls := relaytest.NewMockStats()
for i := 0; i < 10; i++ {
ctx, cancel := NewContext(time.Second)
handler := func(ctx context.Context, args *raw.Args) (*raw.Res, error) {
defer cancel()
return nil, ErrTimeout
}
ts.RegisterFunc("call", handler)
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "call", nil, nil)
assert.Equal(t, err, context.Canceled, "Call should fail due to cancel")
calls.Add(sn, sn, "call").Failed("timeout").End()
}
ts.AssertRelayStats(calls)
})
}
func TestTimeout(t *testing.T) {
testutils.WithTestServer(t, nil, func(ts *testutils.TestServer) {
// onError may be called when the block call tries to write the call response.
onError := func(ctx context.Context, err error) {
assert.Equal(t, ErrTimeout, err, "onError err should be ErrTimeout")
assert.Equal(t, context.DeadlineExceeded, ctx.Err(), "Context should timeout")
}
testHandler := onErrorTestHandler{newTestHandler(t), onError}
ts.Register(raw.Wrap(testHandler), "block")
ctx, cancel := NewContext(testutils.Timeout(15 * time.Millisecond))
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "block", []byte("Arg2"), []byte("Arg3"))
assert.Equal(t, ErrTimeout, err)
// Verify the server-side receives an error from the context.
select {
case err := <-testHandler.blockErr:
assert.Equal(t, context.DeadlineExceeded, err, "Server should have received timeout")
case <-time.After(time.Second):
t.Errorf("Server did not receive call, may need higher timeout")
}
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "block").Failed("timeout").End()
ts.AssertRelayStats(calls)
})
}
func TestRelay(t *testing.T) {
withRelayedEcho(t, func(_, _, client *Channel, ts *testutils.TestServer) {
tests := []struct {
header string
body string
}{
{"fake-header", "fake-body"}, // fits in one frame
{"fake-header", strings.Repeat("fake-body", 10000)}, // requires continuation
}
sc := client.GetSubChannel("test")
for _, tt := range tests {
ctx, cancel := NewContext(time.Second)
defer cancel()
arg2, arg3, _, err := raw.CallSC(ctx, sc, "echo", []byte(tt.header), []byte(tt.body))
require.NoError(t, err, "Relayed call failed.")
assert.Equal(t, tt.header, string(arg2), "Header was mangled during relay.")
assert.Equal(t, tt.body, string(arg3), "Body was mangled during relay.")
}
calls := relaytest.NewMockStats()
peer := relay.Peer{HostPort: ts.Server().PeerInfo().HostPort}
for _ = range tests {
calls.Add("client", "test", "echo").SetPeer(peer).Succeeded().End()
}
ts.AssertRelayStats(calls)
})
}
func TestWriteErrorAfterTimeout(t *testing.T) {
// TODO: Make this test block at different points (e.g. before, during read/write).
testutils.WithTestServer(t, nil, func(ts *testutils.TestServer) {
timedOut := make(chan struct{})
done := make(chan struct{})
handler := func(ctx context.Context, call *InboundCall) {
<-ctx.Done()
<-timedOut
_, err := raw.ReadArgs(call)
assert.Equal(t, ErrTimeout, err, "Read args should fail with timeout")
response := call.Response()
assert.Equal(t, ErrTimeout, response.SendSystemError(ErrServerBusy), "SendSystemError should fail")
close(done)
}
ts.Register(HandlerFunc(handler), "call")
ctx, cancel := NewContext(testutils.Timeout(30 * time.Millisecond))
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "call", nil, testutils.RandBytes(100000))
assert.Equal(t, err, ErrTimeout, "Call should timeout")
close(timedOut)
select {
case <-done:
case <-time.After(time.Second):
t.Errorf("Handler not called, timeout may be too low")
}
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "call").Failed("timeout").End()
ts.AssertRelayStats(calls)
})
}
func TestNoTimeout(t *testing.T) {
testutils.WithTestServer(t, nil, func(ts *testutils.TestServer) {
ts.Register(raw.Wrap(newTestHandler(t)), "Echo")
ctx := context.Background()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), "svc", "Echo", []byte("Headers"), []byte("Body"))
assert.Equal(t, ErrTimeoutRequired, err)
ts.AssertRelayStats(relaytest.NewMockStats())
})
}
func TestNoServiceNaming(t *testing.T) {
testutils.WithTestServer(t, nil, func(ts *testutils.TestServer) {
ctx, cancel := NewContext(time.Second)
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), "", "Echo", []byte("Headers"), []byte("Body"))
assert.Equal(t, ErrNoServiceName, err)
ts.AssertRelayStats(relaytest.NewMockStats())
})
}
func TestLargeTimeout(t *testing.T) {
testutils.WithTestServer(t, nil, func(ts *testutils.TestServer) {
ts.Register(raw.Wrap(newTestHandler(t)), "echo")
ctx, cancel := NewContext(1000 * time.Second)
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "echo", testArg2, testArg3)
assert.NoError(t, err, "Call failed")
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "echo").Succeeded().End()
ts.AssertRelayStats(calls)
})
}
func TestGracefulClose(t *testing.T) {
testutils.WithTestServer(t, nil, func(ts *testutils.TestServer) {
ch2 := ts.NewServer(nil)
hp2 := ch2.PeerInfo().HostPort
defer ch2.Close()
ctx, cancel := NewContext(time.Second)
defer cancel()
assert.NoError(t, ts.Server().Ping(ctx, hp2), "Ping from ch1 -> ch2 failed")
assert.NoError(t, ch2.Ping(ctx, ts.HostPort()), "Ping from ch2 -> ch1 failed")
// No stats for pings.
ts.AssertRelayStats(relaytest.NewMockStats())
})
}
func TestBadRequest(t *testing.T) {
// ch will log an error when it receives a request for an unknown handler.
opts := testutils.NewOpts().AddLogFilter("Couldn't find handler.", 1)
testutils.WithTestServer(t, opts, func(ts *testutils.TestServer) {
ctx, cancel := NewContext(time.Second)
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "Noone", []byte("Headers"), []byte("Body"))
require.NotNil(t, err)
assert.Equal(t, ErrCodeBadRequest, GetSystemErrorCode(err))
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "Noone").Failed("bad-request").End()
ts.AssertRelayStats(calls)
})
}
// NewTestServer constructs a TestServer.
func NewTestServer(t testing.TB, opts *ChannelOpts) *TestServer {
ts := &TestServer{
TB: t,
channelStates: make(map[*tchannel.Channel]*tchannel.RuntimeState),
relayStats: relaytest.NewMockStats(),
introspectOpts: &tchannel.IntrospectionOptions{
IncludeExchanges: true,
IncludeTombstones: true,
},
}
ts.NewServer(opts)
if opts == nil || !opts.DisableRelay {
ts.addRelay(opts)
}
return ts
}
func TestWriteArg3AfterTimeout(t *testing.T) {
// The channel reads and writes during timeouts, causing warning logs.
opts := testutils.NewOpts().DisableLogVerification()
testutils.WithTestServer(t, opts, func(ts *testutils.TestServer) {
timedOut := make(chan struct{})
handler := func(ctx context.Context, call *InboundCall) {
_, err := raw.ReadArgs(call)
assert.NoError(t, err, "Read args failed")
response := call.Response()
assert.NoError(t, NewArgWriter(response.Arg2Writer()).Write(nil), "Write Arg2 failed")
writer, err := response.Arg3Writer()
assert.NoError(t, err, "Arg3Writer failed")
for {
if _, err := writer.Write(testutils.RandBytes(4096)); err != nil {
assert.Equal(t, err, ErrTimeout, "Handler should timeout")
close(timedOut)
return
}
runtime.Gosched()
}
}
ts.Register(HandlerFunc(handler), "call")
ctx, cancel := NewContext(testutils.Timeout(50 * time.Millisecond))
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "call", nil, nil)
assert.Equal(t, err, ErrTimeout, "Call should timeout")
// Wait for the write to complete, make sure there's no errors.
select {
case <-time.After(testutils.Timeout(30 * time.Millisecond)):
t.Errorf("Handler should have failed due to timeout")
case <-timedOut:
}
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "call").Failed("timeout").Succeeded().End()
ts.AssertRelayStats(calls)
})
}
func TestServerBusy(t *testing.T) {
testutils.WithTestServer(t, nil, func(ts *testutils.TestServer) {
ts.Register(ErrorHandlerFunc(func(ctx context.Context, call *InboundCall) error {
if _, err := raw.ReadArgs(call); err != nil {
return err
}
return ErrServerBusy
}), "busy")
ctx, cancel := NewContext(time.Second)
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "busy", []byte("Arg2"), []byte("Arg3"))
require.NotNil(t, err)
assert.Equal(t, ErrCodeBusy, GetSystemErrorCode(err), "err: %v", err)
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "busy").Failed("busy").End()
ts.AssertRelayStats(calls)
})
}
func TestWriteTimeout(t *testing.T) {
testutils.WithTestServer(t, nil, func(ts *testutils.TestServer) {
ch := ts.Server()
ctx, cancel := NewContext(testutils.Timeout(15 * time.Millisecond))
defer cancel()
call, err := ch.BeginCall(ctx, ts.HostPort(), ch.ServiceName(), "call", nil)
require.NoError(t, err, "Call failed")
writer, err := call.Arg2Writer()
require.NoError(t, err, "Arg2Writer failed")
_, err = writer.Write([]byte{1})
require.NoError(t, err, "Write initial bytes failed")
<-ctx.Done()
_, err = io.Copy(writer, testreader.Looper([]byte{1}))
assert.Equal(t, ErrTimeout, err, "Write should fail with timeout")
ts.AssertRelayStats(relaytest.NewMockStats())
})
}
func TestFragmentationSlowReader(t *testing.T) {
// Inbound forward will timeout and cause a warning log.
opts := testutils.NewOpts().
AddLogFilter("Unable to forward frame", 1).
AddLogFilter("Connection error", 1)
testutils.WithTestServer(t, opts, func(ts *testutils.TestServer) {
startReading, handlerComplete := make(chan struct{}), make(chan struct{})
handler := func(ctx context.Context, call *InboundCall) {
<-startReading
<-ctx.Done()
_, err := raw.ReadArgs(call)
assert.Error(t, err, "ReadArgs should fail since frames will be dropped due to slow reading")
close(handlerComplete)
}
ts.Register(HandlerFunc(handler), "echo")
arg2 := testutils.RandBytes(MaxFramePayloadSize * MexChannelBufferSize)
arg3 := testutils.RandBytes(MaxFramePayloadSize * (MexChannelBufferSize + 1))
ctx, cancel := NewContext(testutils.Timeout(30 * time.Millisecond))
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "echo", arg2, arg3)
assert.Error(t, err, "Call should timeout due to slow reader")
close(startReading)
select {
case <-handlerComplete:
case <-time.After(testutils.Timeout(70 * time.Millisecond)):
t.Errorf("Handler not called, context timeout may be too low")
}
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "echo").Failed("timeout").End()
ts.AssertRelayStats(calls)
})
}
func TestErrorFrameEndsRelay(t *testing.T) {
// TestServer validates that there are no relay items left after the given func.
opts := serviceNameOpts("svc").SetRelayOnly().DisableLogVerification()
testutils.WithTestServer(t, opts, func(ts *testutils.TestServer) {
client := ts.NewClient(nil)
err := testutils.CallEcho(client, ts.HostPort(), "svc", nil)
if !assert.Error(t, err, "Expected error due to unknown method") {
return
}
se, ok := err.(SystemError)
if !assert.True(t, ok, "err should be a SystemError, got %T", err) {
return
}
assert.Equal(t, ErrCodeBadRequest, se.Code(), "Expected BadRequest error")
calls := relaytest.NewMockStats()
calls.Add(client.PeerInfo().ServiceName, "svc", "echo").Failed("bad-request").End()
ts.AssertRelayStats(calls)
})
}
func TestRelayConnectionCloseDrainsRelayItems(t *testing.T) {
opts := serviceNameOpts("s1").SetRelayOnly()
testutils.WithTestServer(t, opts, func(ts *testutils.TestServer) {
ctx, cancel := NewContext(time.Second)
defer cancel()
s1 := ts.Server()
s2 := ts.NewServer(serviceNameOpts("s2"))
s2HP := s2.PeerInfo().HostPort
testutils.RegisterEcho(s1, func() {
// When s1 gets called, it calls Close on the connection from the relay to s2.
conn, err := ts.Relay().Peers().GetOrAdd(s2HP).GetConnection(ctx)
require.NoError(t, err, "Unexpected failure getting connection between s1 and relay")
conn.Close()
})
testutils.AssertEcho(t, s2, ts.HostPort(), "s1")
calls := relaytest.NewMockStats()
calls.Add("s2", "s1", "echo").Succeeded().End()
ts.AssertRelayStats(calls)
})
}
func TestUnexpectedHandlerError(t *testing.T) {
opts := testutils.NewOpts().
AddLogFilter("Unexpected handler error", 1)
testutils.WithTestServer(t, opts, func(ts *testutils.TestServer) {
ts.Register(ErrorHandlerFunc(func(ctx context.Context, call *InboundCall) error {
if _, err := raw.ReadArgs(call); err != nil {
return err
}
return fmt.Errorf("nope")
}), "nope")
ctx, cancel := NewContext(time.Second)
defer cancel()
_, _, _, err := raw.Call(ctx, ts.Server(), ts.HostPort(), ts.ServiceName(), "nope", []byte("Arg2"), []byte("Arg3"))
require.NotNil(t, err)
assert.Equal(t, ErrCodeUnexpected, GetSystemErrorCode(err), "err: %v", err)
calls := relaytest.NewMockStats()
calls.Add(ts.ServiceName(), ts.ServiceName(), "nope").Failed("unexpected-error").End()
ts.AssertRelayStats(calls)
})
}
func TestRelayErrorsOnGetPeer(t *testing.T) {
busyErr := NewSystemError(ErrCodeBusy, "busy")
tests := []struct {
desc string
addPeer func(*testutils.SimpleRelayHosts)
statsKey string
statsPeer relay.Peer
wantErr error
}{
{
desc: "No peer mappings, return empty Peer",
addPeer: func(_ *testutils.SimpleRelayHosts) {},
statsKey: "relay-declined",
wantErr: NewSystemError(ErrCodeDeclined, `invalid peer for "svc"`),
},
{
desc: "System error getting peer",
addPeer: func(rh *testutils.SimpleRelayHosts) {
rh.AddError("svc", busyErr)
},
statsKey: "relay-busy",
wantErr: busyErr,
},
{
desc: "Unknown error getting peer",
addPeer: func(rh *testutils.SimpleRelayHosts) {
rh.AddError("svc", errors.New("unknown"))
},
statsKey: "relay-declined",
wantErr: NewSystemError(ErrCodeDeclined, "unknown"),
},
{
desc: "No host:port on peer",
addPeer: func(rh *testutils.SimpleRelayHosts) {
rh.AddPeer("svc", "", "pool", "zone")
},
statsKey: "relay-declined",
statsPeer: relay.Peer{Zone: "zone", Pool: "pool"},
wantErr: NewSystemError(ErrCodeDeclined, `invalid peer for "svc"`),
},
}
for _, tt := range tests {
opts := testutils.NewOpts().SetRelayOnly()
testutils.WithTestServer(t, opts, func(ts *testutils.TestServer) {
tt.addPeer(ts.RelayHosts())
client := ts.NewClient(nil)
err := testutils.CallEcho(client, ts.HostPort(), "svc", nil)
if !assert.Error(t, err, "Call to unknown service should fail") {
return
}
assert.Equal(t, tt.wantErr, err, "%v: unexpected error", tt.desc)
calls := relaytest.NewMockStats()
calls.Add(client.PeerInfo().ServiceName, "svc", "echo").
SetPeer(tt.statsPeer).
Failed(tt.statsKey).End()
ts.AssertRelayStats(calls)
})
}
}
