func makeTestDBClient(t *testing.T, s *server.TestServer) *sql.DB {
db, err := sql.Open("cockroach", fmt.Sprintf("https://%s@%s?certs=test_certs",
security.RootUser,
s.ServingAddr()))
if err != nil {
t.Fatal(err)
}
return db
}
// setupMultipleRanges creates a test server and splits the
// key range at the given keys. Returns the test server and client.
// The caller is responsible for stopping the server and
// closing the client.
func setupMultipleRanges(t *testing.T, ts *server.TestServer, splitAt ...string) *client.DB {
db := createTestClient(t, ts.Stopper(), ts.ServingAddr())
// Split the keyspace at the given keys.
for _, key := range splitAt {
if err := db.AdminSplit(key); err != nil {
// Don't leak server goroutines.
t.Fatal(err)
}
}
return db
}
func initReverseScanTestEnv(s *server.TestServer, t *testing.T) *client.DB {
db := createTestClient(t, s.Stopper(), s.ServingAddr())
// Set up multiple ranges:
// ["", "b"),["b", "e") ,["e", "g") and ["g", "\xff\xff").
for _, key := range []string{"b", "e", "g"} {
// Split the keyspace at the given key.
if pErr := db.AdminSplit(key); pErr != nil {
t.Fatal(pErr)
}
}
// Write keys before, at, and after the split key.
for _, key := range []string{"a", "b", "c", "d", "e", "f", "g", "h"} {
if pErr := db.Put(key, "value"); pErr != nil {
t.Fatal(pErr)
}
}
return db
}
// PGUrl returns a postgres connection url which connects to this server with the given user, and a
// cleanup function which must be called after all connections created using the connection url have
// been closed.
//
// In order to connect securely using postgres, this method will create temporary on-disk copies of
// certain embedded security certificates. The certificates will be created in a new temporary
// directory. The returned cleanup function will delete this temporary directory.
// Note that two calls to this function for the same `user` will generate different
// copies of the certificates, so the cleanup function must always be called.
//
// Args:
// prefix: A prefix to be prepended to the temp file names generated, for debugging.
func PGUrl(t testing.TB, ts *server.TestServer, user, prefix string) (url.URL, func()) {
host, port, err := net.SplitHostPort(ts.ServingAddr())
if err != nil {
t.Fatal(err)
}
tempDir, err := ioutil.TempDir("", prefix)
if err != nil {
t.Fatal(err)
}
caPath := filepath.Join(security.EmbeddedCertsDir, security.EmbeddedCACert)
certPath := filepath.Join(security.EmbeddedCertsDir, fmt.Sprintf("%s.crt", user))
keyPath := filepath.Join(security.EmbeddedCertsDir, fmt.Sprintf("%s.key", user))
// Copy these assets to disk from embedded strings, so this test can
// run from a standalone binary.
tempCAPath := securitytest.RestrictedCopy(t, caPath, tempDir, "ca")
tempCertPath := securitytest.RestrictedCopy(t, certPath, tempDir, "cert")
tempKeyPath := securitytest.RestrictedCopy(t, keyPath, tempDir, "key")
options := url.Values{}
options.Add("sslmode", "verify-full")
options.Add("sslrootcert", tempCAPath)
options.Add("sslcert", tempCertPath)
options.Add("sslkey", tempKeyPath)
return url.URL{
Scheme: "postgres",
User: url.User(user),
Host: net.JoinHostPort(host, port),
RawQuery: options.Encode(),
}, func() {
if err := os.RemoveAll(tempDir); err != nil {
// Not Fatal() because we might already be panicking.
t.Error(err)
}
}
}
// TestRequestToUninitializedRange tests the behavior when a request
// is sent to a node which should be a replica of the correct range
// but has not yet received its initial snapshot. This would
// previously panic due to a malformed error response from the server,
// as seen in https://github.com/cockroachdb/cockroach/issues/6027.
//
// Prior to the other changes in the commit that introduced it, this
// test would reliable trigger the panic from #6027. However, it
// relies on some hacky tricks to both trigger the panic and shut down
// cleanly. If this test needs a lot of maintenance in the future we
// should be willing to get rid of it.
func TestRequestToUninitializedRange(t *testing.T) {
defer leaktest.AfterTest(t)()
s := server.TestServer{StoresPerNode: 2}
if err := s.Start(); err != nil {
t.Fatalf("Could not start server: %v", err)
}
defer s.Stop()
// Choose a range ID that is much larger than any that would be
// created by initial splits.
const rangeID = roachpb.RangeID(1000)
// Set up a range with replicas on two stores of the same node. This
// ensures that the DistSender will consider both replicas healthy
// and will try to talk to both (so we can get a non-retryable error
// from the second store).
replica1 := roachpb.ReplicaDescriptor{
NodeID: 1,
StoreID: 1,
ReplicaID: 1,
}
replica2 := roachpb.ReplicaDescriptor{
NodeID: 1,
StoreID: 2,
ReplicaID: 2,
}
// HACK: remove the second store from the node to generate a
// non-retryable error when we try to talk to it.
store2, err := s.Stores().GetStore(2)
if err != nil {
t.Fatal(err)
}
s.Stores().RemoveStore(store2)
// Create the uninitialized range by sending an isolated raft
// message to the first store.
conn, err := s.RPCContext().GRPCDial(s.ServingAddr())
if err != nil {
t.Fatal(err)
}
raftClient := storage.NewMultiRaftClient(conn)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
stream, err := raftClient.RaftMessage(ctx)
if err != nil {
t.Fatal(err)
}
msg := storage.RaftMessageRequest{
GroupID: rangeID,
ToReplica: replica1,
FromReplica: replica2,
Message: raftpb.Message{
Type: raftpb.MsgApp,
To: 1,
},
}
if err := stream.Send(&msg); err != nil {
t.Fatal(err)
}
// Make sure the replica was created.
store1, err := s.Stores().GetStore(1)
if err != nil {
t.Fatal(err)
}
util.SucceedsSoon(t, func() error {
if replica, err := store1.GetReplica(rangeID); err != nil {
return util.Errorf("failed to look up replica: %s", err)
} else if replica.IsInitialized() {
return util.Errorf("expected replica to be uninitialized")
}
return nil
})
// Create our own DistSender so we can force some requests to the
// bogus range. The DistSender needs to be in scope for its own
// MockRangeDescriptorDB closure.
var sender *kv.DistSender
sender = kv.NewDistSender(&kv.DistSenderContext{
Clock: s.Clock(),
RPCContext: s.RPCContext(),
RangeDescriptorDB: kv.MockRangeDescriptorDB(
func(key roachpb.RKey, considerIntents, useReverseScan bool,
) ([]roachpb.RangeDescriptor, []roachpb.RangeDescriptor, *roachpb.Error) {
if key.Equal(roachpb.RKeyMin) {
// Pass through requests for the first range to the real sender.
desc, err := sender.FirstRange()
if err != nil {
return nil, nil, roachpb.NewError(err)
}
return []roachpb.RangeDescriptor{*desc}, nil, nil
}
return []roachpb.RangeDescriptor{{
RangeID: rangeID,
StartKey: roachpb.RKey(keys.Meta2Prefix),
EndKey: roachpb.RKeyMax,
Replicas: []roachpb.ReplicaDescriptor{replica1, replica2},
}}, nil, nil
}),
}, s.Gossip())
// Only inconsistent reads triggered the panic in #6027.
hdr := roachpb.Header{
ReadConsistency: roachpb.INCONSISTENT,
}
req := roachpb.NewGet(roachpb.Key("asdf"))
// Repeat the test a few times: due to the randomization between the
// two replicas, each attempt only had a 50% chance of triggering
// the panic.
for i := 0; i < 5; i++ {
_, pErr := client.SendWrappedWith(sender, context.Background(), hdr, req)
// Each attempt fails with "store 2 not found" because that is the
// non-retryable error.
if !testutils.IsPError(pErr, "store 2 not found") {
t.Fatal(pErr)
}
}
}