// Start starts the test cluster by bootstrapping an in-memory store
// (defaults to maximum of 50M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.Addr after Start() for client connections. Use Stop()
// to shutdown the server after the test completes.
func (ltc *LocalTestCluster) Start(t util.Tester) {
ltc.Manual = hlc.NewManualClock(0)
ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano)
ltc.Stopper = util.NewStopper()
rpcContext := rpc.NewContext(testutils.NewTestBaseContext(), ltc.Clock, ltc.Stopper)
ltc.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap)
ltc.Eng = engine.NewInMem(proto.Attributes{}, 50<<20)
ltc.lSender = newRetryableLocalSender(NewLocalSender())
ltc.Sender = NewTxnCoordSender(ltc.lSender, ltc.Clock, false, ltc.Stopper)
var err error
if ltc.DB, err = client.Open("//root@", client.SenderOpt(ltc.Sender)); err != nil {
t.Fatal(err)
}
transport := multiraft.NewLocalRPCTransport()
ltc.Stopper.AddCloser(transport)
ctx := storage.TestStoreContext
ctx.Clock = ltc.Clock
ctx.DB = ltc.DB
ctx.Gossip = ltc.Gossip
ctx.Transport = transport
ltc.Store = storage.NewStore(ctx, ltc.Eng, &proto.NodeDescriptor{NodeID: 1})
if err := ltc.Store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.lSender.AddStore(ltc.Store)
if err := ltc.Store.BootstrapRange(); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
if err := ltc.Store.Start(ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
}
// getText fetches the HTTP response body as text in the form of a
// byte slice from the specified URL.
func getText(url string) ([]byte, error) {
// There are no particular permissions on admin endpoints, TestUser is fine.
client, err := testutils.NewTestBaseContext(TestUser).GetHTTPClient()
if err != nil {
return nil, err
}
resp, err := client.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return ioutil.ReadAll(resp.Body)
}
// TestHTTPSenderRetryHTTPSendError verifies that send is retried
// on all errors sending HTTP requests.
func TestHTTPSenderRetryHTTPSendError(t *testing.T) {
defer leaktest.AfterTest(t)
retryOptions := defaultRetryOptions
retryOptions.Backoff = 1 * time.Millisecond
testCases := []func(*httptest.Server, http.ResponseWriter){
// Send back an unparseable response but a success code on first try.
func(s *httptest.Server, w http.ResponseWriter) {
fmt.Fprintf(w, "\xff\xfe\x23\x44")
},
// Close the client connection.
func(s *httptest.Server, w http.ResponseWriter) {
s.CloseClientConnections()
},
}
for i, testFunc := range testCases {
count := 0
var s *httptest.Server
server, addr := startTestHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
count++
if count == 1 {
// On first retry, invoke the error function.
testFunc(s, w)
return
}
// Success on second try.
body, contentType, err := util.MarshalResponse(r, testPutResp, util.AllEncodings)
if err != nil {
t.Errorf("%d: failed to marshal response: %s", i, err)
}
w.Header().Set(util.ContentTypeHeader, contentType)
w.Write(body)
}))
s = server
sender, err := newHTTPSender(addr, testutils.NewTestBaseContext(), retryOptions)
if err != nil {
t.Fatal(err)
}
reply := &proto.PutResponse{}
sender.Send(context.Background(), proto.Call{Args: testPutReq, Reply: reply})
if reply.GoError() != nil {
t.Errorf("%d: expected success; got %s", i, reply.GoError())
}
if count != 2 {
t.Errorf("%d: expected retry", i)
}
server.Close()
}
}
// TestHTTPSenderSend verifies sending posts.
func TestHTTPSenderSend(t *testing.T) {
defer leaktest.AfterTest(t)
server, addr := startTestHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
t.Errorf("expected method POST; got %s", r.Method)
}
if r.URL.Path != KVDBEndpoint+"Put" {
t.Errorf("expected url %s; got %s", KVDBEndpoint+"Put", r.URL.Path)
}
// Unmarshal the request.
reqBody, err := ioutil.ReadAll(r.Body)
if err != nil {
t.Errorf("unexpected error reading body: %s", err)
}
args := &proto.PutRequest{}
if err := util.UnmarshalRequest(r, reqBody, args, util.AllEncodings); err != nil {
t.Errorf("unexpected error unmarshalling request: %s", err)
}
if !args.Key.Equal(testPutReq.Key) || !args.Timestamp.Equal(testPutReq.Timestamp) {
t.Errorf("expected parsed %+v to equal %+v", args, testPutReq)
}
body, contentType, err := util.MarshalResponse(r, testPutResp, util.AllEncodings)
if err != nil {
t.Errorf("failed to marshal response: %s", err)
}
w.Header().Set(util.ContentTypeHeader, contentType)
w.Write(body)
}))
defer server.Close()
sender, err := newHTTPSender(addr, testutils.NewTestBaseContext(), defaultRetryOptions)
if err != nil {
t.Fatal(err)
}
reply := &proto.PutResponse{}
sender.Send(context.Background(), proto.Call{Args: testPutReq, Reply: reply})
if reply.GoError() != nil {
t.Errorf("expected success; got %s", reply.GoError())
}
if !reply.Timestamp.Equal(testPutResp.Timestamp) {
t.Errorf("expected received %+v to equal %+v", reply, testPutResp)
}
}
// TestAdminDebugRedirect verifies that the /debug/ endpoint is redirected to on
// incorrect /debug/ paths.
func TestAdminDebugRedirect(t *testing.T) {
defer leaktest.AfterTest(t)()
s := StartTestServer(t)
defer s.Stop()
expURL := debugURL(s)
origURL := expURL + "incorrect"
// There are no particular permissions on admin endpoints, TestUser is fine.
client, err := testutils.NewTestBaseContext(TestUser).GetHTTPClient()
if err != nil {
t.Fatal(err)
}
// Don't follow redirects automatically.
redirectAttemptedError := errors.New("redirect")
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return redirectAttemptedError
}
resp, err := client.Get(origURL)
if urlError, ok := err.(*url.Error); ok && urlError.Err == redirectAttemptedError {
// Ignore the redirectAttemptedError.
err = nil
}
if err != nil {
t.Fatal(err)
} else {
resp.Body.Close()
if resp.StatusCode != http.StatusMovedPermanently {
t.Errorf("expected status code %d; got %d", http.StatusMovedPermanently, resp.StatusCode)
}
if redirectURL, err := resp.Location(); err != nil {
t.Error(err)
} else if foundURL := redirectURL.String(); foundURL != expURL {
t.Errorf("expected location %s; got %s", expURL, foundURL)
}
}
}
"golang.org/x/net/context"
snappy "github.com/cockroachdb/c-snappy"
"github.com/cockroachdb/cockroach/kv"
"github.com/cockroachdb/cockroach/proto"
"github.com/cockroachdb/cockroach/storage"
"github.com/cockroachdb/cockroach/storage/engine"
"github.com/cockroachdb/cockroach/testutils"
"github.com/cockroachdb/cockroach/util"
"github.com/cockroachdb/cockroach/util/leaktest"
"github.com/cockroachdb/cockroach/util/log"
)
var testContext = NewTestContext()
var testBaseContext = testutils.NewTestBaseContext()
var serverTestBaseContext = testutils.NewServerTestBaseContext()
// createTestConfigFile creates a temporary file and writes the
// testConfig yaml data to it. The caller is responsible for
// removing it. Returns the filename for a subsequent call to
// os.Remove().
func createTestConfigFile(body string) string {
f, err := ioutil.TempFile("", "test-config")
if err != nil {
log.Fatalf("failed to open temporary file: %v", err)
}
defer f.Close()
if _, err = f.Write([]byte(body)); err != nil {
log.Fatalf("failed to write to temporary file: %v", err)
}
// Verify client certificate enforcement and user whitelisting.
func TestSSLEnforcement(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
// HTTPS with client certs for security.RootUser.
rootCertsContext := testutils.NewTestBaseContext(security.RootUser)
// HTTPS with client certs for security.NodeUser.
nodeCertsContext := testutils.NewNodeTestBaseContext()
// HTTPS with client certs for TestUser.
testCertsContext := testutils.NewTestBaseContext(TestUser)
// HTTPS without client certs. The user does not matter.
noCertsContext := testutils.NewTestBaseContext(TestUser)
noCertsContext.SSLCert = ""
// Plain http.
insecureContext := testutils.NewTestBaseContext(TestUser)
insecureContext.Insecure = true
kvGet := &roachpb.GetRequest{}
kvGet.Key = roachpb.Key("/")
testCases := []struct {
method, path string
body proto.Message
ctx *base.Context
success bool // request sent successfully (may be non-200)
code int // http response code
}{
// /ui/: basic file server: no auth.
{"GET", "/index.html", nil, rootCertsContext, true, http.StatusOK},
{"GET", "/index.html", nil, nodeCertsContext, true, http.StatusOK},
{"GET", "/index.html", nil, testCertsContext, true, http.StatusOK},
{"GET", "/index.html", nil, noCertsContext, true, http.StatusOK},
// TODO(tamird): s/308/http.StatusPermanentRedirect/ when it exists.
{"GET", "/index.html", nil, insecureContext, true, 308},
// /_admin/: server.adminServer: no auth.
{"GET", healthPath, nil, rootCertsContext, true, http.StatusOK},
{"GET", healthPath, nil, nodeCertsContext, true, http.StatusOK},
{"GET", healthPath, nil, testCertsContext, true, http.StatusOK},
{"GET", healthPath, nil, noCertsContext, true, http.StatusOK},
// TODO(tamird): s/308/http.StatusPermanentRedirect/ when it exists.
{"GET", healthPath, nil, insecureContext, true, 308},
// /debug/: server.adminServer: no auth.
{"GET", debugEndpoint + "vars", nil, rootCertsContext, true, http.StatusOK},
{"GET", debugEndpoint + "vars", nil, nodeCertsContext, true, http.StatusOK},
{"GET", debugEndpoint + "vars", nil, testCertsContext, true, http.StatusOK},
{"GET", debugEndpoint + "vars", nil, noCertsContext, true, http.StatusOK},
// TODO(tamird): s/308/http.StatusPermanentRedirect/ when it exists.
{"GET", debugEndpoint + "vars", nil, insecureContext, true, 308},
// /_status/nodes: server.statusServer: no auth.
{"GET", statusNodesPrefix, nil, rootCertsContext, true, http.StatusOK},
{"GET", statusNodesPrefix, nil, nodeCertsContext, true, http.StatusOK},
{"GET", statusNodesPrefix, nil, testCertsContext, true, http.StatusOK},
{"GET", statusNodesPrefix, nil, noCertsContext, true, http.StatusOK},
// TODO(tamird): s/308/http.StatusPermanentRedirect/ when it exists.
{"GET", statusNodesPrefix, nil, insecureContext, true, 308},
// /ts/: ts.Server: no auth.
{"GET", ts.URLPrefix, nil, rootCertsContext, true, http.StatusNotFound},
{"GET", ts.URLPrefix, nil, nodeCertsContext, true, http.StatusNotFound},
{"GET", ts.URLPrefix, nil, testCertsContext, true, http.StatusNotFound},
{"GET", ts.URLPrefix, nil, noCertsContext, true, http.StatusNotFound},
// TODO(tamird): s/308/http.StatusPermanentRedirect/ when it exists.
{"GET", ts.URLPrefix, nil, insecureContext, true, 308},
}
for tcNum, tc := range testCases {
client, err := tc.ctx.GetHTTPClient()
if err != nil {
t.Fatalf("[%d]: failed to get http client: %v", tcNum, err)
}
url := fmt.Sprintf(
"%s://%s%s", tc.ctx.HTTPRequestScheme(),
s.(*TestServer).Ctx.HTTPAddr, tc.path)
resp, err := doHTTPReq(t, client, tc.method, url, tc.body)
if (err == nil) != tc.success {
t.Errorf("[%d]: expected success=%t, got err=%v", tcNum, tc.success, err)
}
if err != nil {
continue
}
defer resp.Body.Close()
if resp.StatusCode != tc.code {
t.Errorf("[%d]: expected status code %d, got %d", tcNum, tc.code, resp.StatusCode)
}
}
}
// Verify client certificate enforcement and user whitelisting.
func TestSSLEnforcement(t *testing.T) {
defer leaktest.AfterTest(t)
s := StartTestServer(t)
defer s.Stop()
// HTTPS with client certs for "root".
rootCertsContext := testutils.NewTestBaseContext(security.RootUser)
// HTTPS with client certs for "node".
nodeCertsContext := testutils.NewNodeTestBaseContext()
// HTTPS with client certs for testuser.
testCertsContext := testutils.NewTestBaseContext(TestUser)
// HTTPS without client certs. The user does not matter.
noCertsContext := testutils.NewTestBaseContext(TestUser)
noCertsContext.Certs = ""
// Plain http.
insecureContext := testutils.NewTestBaseContext(TestUser)
insecureContext.Insecure = true
kvGet := &roachpb.GetRequest{}
kvGet.Key = roachpb.Key("/")
testCases := []struct {
method, key string
body proto.Message
ctx *base.Context
success bool // request sent successfully (may be non-200)
code int // http response code
}{
// /ui/: basic file server: no auth.
{"GET", "/index.html", nil, rootCertsContext, true, http.StatusOK},
{"GET", "/index.html", nil, nodeCertsContext, true, http.StatusOK},
{"GET", "/index.html", nil, testCertsContext, true, http.StatusOK},
{"GET", "/index.html", nil, noCertsContext, true, http.StatusOK},
{"GET", "/index.html", nil, insecureContext, false, -1},
// /_admin/: server.adminServer: no auth.
{"GET", healthPath, nil, rootCertsContext, true, http.StatusOK},
{"GET", healthPath, nil, nodeCertsContext, true, http.StatusOK},
{"GET", healthPath, nil, testCertsContext, true, http.StatusOK},
{"GET", healthPath, nil, noCertsContext, true, http.StatusOK},
{"GET", healthPath, nil, insecureContext, false, -1},
// /debug/: server.adminServer: no auth.
{"GET", debugEndpoint + "vars", nil, rootCertsContext, true, http.StatusOK},
{"GET", debugEndpoint + "vars", nil, nodeCertsContext, true, http.StatusOK},
{"GET", debugEndpoint + "vars", nil, testCertsContext, true, http.StatusOK},
{"GET", debugEndpoint + "vars", nil, noCertsContext, true, http.StatusOK},
{"GET", debugEndpoint + "vars", nil, insecureContext, false, -1},
// /_status/nodes: server.statusServer: no auth.
{"GET", statusNodesPrefix, nil, rootCertsContext, true, http.StatusOK},
{"GET", statusNodesPrefix, nil, nodeCertsContext, true, http.StatusOK},
{"GET", statusNodesPrefix, nil, testCertsContext, true, http.StatusOK},
{"GET", statusNodesPrefix, nil, noCertsContext, true, http.StatusOK},
{"GET", statusNodesPrefix, nil, insecureContext, false, -1},
// /ts/: ts.Server: no auth.
{"GET", ts.URLPrefix, nil, rootCertsContext, true, http.StatusNotFound},
{"GET", ts.URLPrefix, nil, nodeCertsContext, true, http.StatusNotFound},
{"GET", ts.URLPrefix, nil, testCertsContext, true, http.StatusNotFound},
{"GET", ts.URLPrefix, nil, noCertsContext, true, http.StatusNotFound},
{"GET", ts.URLPrefix, nil, insecureContext, false, -1},
// /sql/: sql.Server. These are proto reqs. The important field is header.User.
{"POST", driver.Endpoint + driver.Execute.String(), sqlForUser(rootCertsContext),
rootCertsContext, true, http.StatusOK},
{"POST", driver.Endpoint + driver.Execute.String(), sqlForUser(nodeCertsContext),
nodeCertsContext, true, http.StatusOK},
{"POST", driver.Endpoint + driver.Execute.String(), sqlForUser(testCertsContext),
testCertsContext, true, http.StatusOK},
{"POST", driver.Endpoint + driver.Execute.String(), sqlForUser(noCertsContext),
noCertsContext, true, http.StatusUnauthorized},
{"POST", driver.Endpoint + driver.Execute.String(), sqlForUser(insecureContext),
insecureContext, false, -1},
}
for tcNum, tc := range testCases {
client, err := tc.ctx.GetHTTPClient()
if err != nil {
t.Fatalf("[%d]: failed to get http client: %v", tcNum, err)
}
resp, err := doHTTPReq(t, client, tc.method,
fmt.Sprintf("%s://%s%s", tc.ctx.HTTPRequestScheme(), s.ServingAddr(), tc.key),
tc.body)
if (err == nil) != tc.success {
t.Fatalf("[%d]: expected success=%t, got err=%v", tcNum, tc.success, err)
}
if err != nil {
continue
}
defer resp.Body.Close()
if resp.StatusCode != tc.code {
t.Errorf("[%d]: expected status code %d, got %d", tcNum, tc.code, resp.StatusCode)
}
}
}
// Verify client certificate enforcement.
func TestSSLEnforcement(t *testing.T) {
defer leaktest.AfterTest(t)
s := StartTestServer(t)
defer s.Stop()
testCases := []struct {
method, key string
certsStatus int // Status code for https with client certs.
noCertsStatus int // Status code for https without client certs.
}{
// /ui/: basic file server: no auth.
{"GET", "/index.html", http.StatusOK, http.StatusOK},
// /_admin/: server.adminServer: no auth.
{"GET", healthPath, http.StatusOK, http.StatusOK},
// /debug/: server.adminServer: no auth.
{"GET", debugEndpoint + "vars", http.StatusOK, http.StatusOK},
// /_status/nodes: server.statusServer: no auth.
{"GET", statusNodeKeyPrefix, http.StatusOK, http.StatusOK},
// /kv/db/: kv.DBServer. These are proto reqs, but we can at least get past auth.
{"GET", kv.DBPrefix + "Get", http.StatusBadRequest, http.StatusUnauthorized},
// /ts/: ts.Server.
{"GET", ts.URLPrefix, http.StatusNotFound, http.StatusUnauthorized},
}
// HTTPS with client certs.
certsContext := testutils.NewTestBaseContext()
client, err := certsContext.GetHTTPClient()
if err != nil {
t.Fatalf("error initializing http client: %s", err)
}
for tcNum, tc := range testCases {
resp, err := doHTTPReq(t, client, tc.method,
fmt.Sprintf("%s://%s%s", certsContext.RequestScheme(), s.ServingAddr(), tc.key))
if err != nil {
t.Fatalf("[%d]: error issuing request: %s", tcNum, err)
}
defer resp.Body.Close()
if resp.StatusCode != tc.certsStatus {
t.Errorf("[%d]: expected status code %d, got %d", tcNum, tc.certsStatus, resp.StatusCode)
}
}
// HTTPS without client certs.
noCertsContext := testutils.NewTestBaseContext()
noCertsContext.Certs = ""
client, err = noCertsContext.GetHTTPClient()
if err != nil {
t.Fatalf("error initializing http client: %s", err)
}
for tcNum, tc := range testCases {
resp, err := doHTTPReq(t, client, tc.method,
fmt.Sprintf("%s://%s%s", noCertsContext.RequestScheme(), s.ServingAddr(), tc.key))
if err != nil {
t.Fatalf("[%d]: error issuing request: %s", tcNum, err)
}
defer resp.Body.Close()
if resp.StatusCode != tc.noCertsStatus {
t.Errorf("[%d]: expected status code %d, got %d", tcNum, tc.noCertsStatus, resp.StatusCode)
}
}
// Plain http.
insecureContext := testutils.NewTestBaseContext()
insecureContext.Insecure = true
client, err = insecureContext.GetHTTPClient()
if err != nil {
t.Fatalf("error initializing http client: %s", err)
}
for tcNum, tc := range testCases {
resp, err := doHTTPReq(t, client, tc.method,
fmt.Sprintf("%s://%s%s", insecureContext.RequestScheme(), s.ServingAddr(), tc.key))
// We're talking http to a https server. We don't even make it to a response.
if err == nil {
defer resp.Body.Close()
t.Errorf("[%d]: unexpected success", tcNum)
}
}
}
// TestHTTPSenderRetryResponseCodes verifies that send is retried
// on some HTTP response codes but not on others.
func TestHTTPSenderRetryResponseCodes(t *testing.T) {
defer leaktest.AfterTest(t)
retryOptions := defaultRetryOptions
retryOptions.Backoff = 1 * time.Millisecond
testCases := []struct {
code int
retry bool
}{
{http.StatusServiceUnavailable, true},
{http.StatusGatewayTimeout, true},
{StatusTooManyRequests, true},
{http.StatusRequestTimeout, false},
{http.StatusBadRequest, false},
{http.StatusNotFound, false},
{http.StatusUnauthorized, false},
{http.StatusForbidden, false},
{http.StatusMethodNotAllowed, false},
{http.StatusNotAcceptable, false},
{http.StatusInternalServerError, false},
{http.StatusNotImplemented, false},
}
for i, test := range testCases {
count := 0
server, addr := startTestHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
count++
if count == 1 {
http.Error(w, "manufactured error", test.code)
return
}
if !test.retry {
t.Errorf("%d: didn't expect retry on code %d", i, test.code)
}
body, contentType, err := util.MarshalResponse(r, testPutResp, util.AllEncodings)
if err != nil {
t.Errorf("%d: failed to marshal response: %s", i, err)
}
w.Header().Set(util.ContentTypeHeader, contentType)
w.Write(body)
}))
sender, err := newHTTPSender(addr, testutils.NewTestBaseContext(), retryOptions)
if err != nil {
t.Fatal(err)
}
reply := &proto.PutResponse{}
sender.Send(context.Background(), proto.Call{Args: testPutReq, Reply: reply})
if test.retry {
if count != 2 {
t.Errorf("%d: expected retry", i)
}
if reply.GoError() != nil {
t.Errorf("%d: expected success after retry; got %s", i, reply.GoError())
}
} else {
if count != 1 {
t.Errorf("%d; expected no retry; got %d", i, count)
}
if reply.GoError() == nil {
t.Errorf("%d: expected error", i)
}
}
server.Close()
}
}