func countRangeReplicas(db *client.DB) (int, error) {
desc := &roachpb.RangeDescriptor{}
if err := db.GetProto(context.TODO(), keys.RangeDescriptorKey(roachpb.RKeyMin), desc); err != nil {
return 0, err
}
return len(desc.Replicas), nil
}
func (tc *TestCluster) changeReplicas(
action roachpb.ReplicaChangeType, startKey roachpb.RKey, targets ...ReplicationTarget,
) (*roachpb.RangeDescriptor, error) {
rangeDesc := &roachpb.RangeDescriptor{}
// TODO(andrei): the following code has been adapted from
// multiTestContext.replicateRange(). Find a way to share.
for _, target := range targets {
// Perform a consistent read to get the updated range descriptor (as opposed
// to just going to one of the stores), to make sure we have the effects of
// the previous ChangeReplicas call. By the time ChangeReplicas returns the
// raft leader is guaranteed to have the updated version, but followers are
// not.
if err := tc.Servers[0].DB().GetProto(context.TODO(),
keys.RangeDescriptorKey(startKey), rangeDesc); err != nil {
return nil, err
}
// Ask an arbitrary replica of the range to perform the change. Note that
// the target for addition/removal is specified, this is about the choice
// of which replica receives the ChangeReplicas operation.
store, err := tc.findMemberStore(rangeDesc.Replicas[0].StoreID)
if err != nil {
return nil, err
}
replica, err := store.GetReplica(rangeDesc.RangeID)
if err != nil {
return nil, err
}
ctx := replica.AnnotateCtx(context.Background())
if err := replica.ChangeReplicas(
ctx,
action,
roachpb.ReplicaDescriptor{
NodeID: target.NodeID,
StoreID: target.StoreID,
}, rangeDesc,
); err != nil {
return nil, err
}
}
if err := tc.Servers[0].DB().GetProto(context.TODO(),
keys.RangeDescriptorKey(startKey), rangeDesc); err != nil {
return nil, err
}
return rangeDesc, nil
}
// SplitRange splits the range containing splitKey.
// The right range created by the split starts at the split key and extends to the
// original range's end key.
// Returns the new descriptors of the left and right ranges.
//
// splitKey must correspond to a SQL table key (it must end with a family ID /
// col ID).
func (ts *TestServer) SplitRange(
splitKey roachpb.Key,
) (roachpb.RangeDescriptor, roachpb.RangeDescriptor, error) {
splitRKey, err := keys.Addr(splitKey)
if err != nil {
return roachpb.RangeDescriptor{}, roachpb.RangeDescriptor{}, err
}
origRangeDesc, err := ts.LookupRange(splitKey)
if err != nil {
return roachpb.RangeDescriptor{}, roachpb.RangeDescriptor{}, err
}
if origRangeDesc.StartKey.Equal(splitRKey) {
return roachpb.RangeDescriptor{}, roachpb.RangeDescriptor{},
errors.Errorf(
"cannot split range %+v at start key %q", origRangeDesc, splitKey)
}
splitReq := roachpb.AdminSplitRequest{
Span: roachpb.Span{
Key: splitKey,
},
SplitKey: splitKey,
}
_, pErr := client.SendWrapped(context.Background(), ts.DistSender(), &splitReq)
if pErr != nil {
return roachpb.RangeDescriptor{}, roachpb.RangeDescriptor{},
errors.Errorf(
"%q: split unexpected error: %s", splitReq.SplitKey, pErr)
}
var leftRangeDesc, rightRangeDesc roachpb.RangeDescriptor
if err := ts.DB().GetProto(context.TODO(),
keys.RangeDescriptorKey(origRangeDesc.StartKey), &leftRangeDesc); err != nil {
return roachpb.RangeDescriptor{}, roachpb.RangeDescriptor{},
errors.Wrap(err, "could not look up left-hand side descriptor")
}
// The split point might not be exactly the one we requested (it can be
// adjusted slightly so we don't split in the middle of SQL rows). Update it
// to the real point.
splitRKey = leftRangeDesc.EndKey
if err := ts.DB().GetProto(context.TODO(),
keys.RangeDescriptorKey(splitRKey), &rightRangeDesc); err != nil {
return roachpb.RangeDescriptor{}, roachpb.RangeDescriptor{},
errors.Wrap(err, "could not look up right-hand side descriptor")
}
return leftRangeDesc, rightRangeDesc, nil
}
// TestBatchPrevNext tests batch.{Prev,Next}.
func TestBatchPrevNext(t *testing.T) {
defer leaktest.AfterTest(t)()
loc := func(s string) string {
return string(keys.RangeDescriptorKey(roachpb.RKey(s)))
}
span := func(strs ...string) []roachpb.Span {
var r []roachpb.Span
for i, str := range strs {
if i%2 == 0 {
r = append(r, roachpb.Span{Key: roachpb.Key(str)})
} else {
r[len(r)-1].EndKey = roachpb.Key(str)
}
}
return r
}
max, min := string(roachpb.RKeyMax), string(roachpb.RKeyMin)
abc := span("a", "", "b", "", "c", "")
testCases := []struct {
spans []roachpb.Span
key, expFW, expBW string
}{
{spans: span("a", "c", "b", ""), key: "b", expFW: "b", expBW: "b"},
{spans: span("a", "c", "b", ""), key: "a", expFW: "a", expBW: "a"},
{spans: span("a", "c", "d", ""), key: "c", expFW: "d", expBW: "c"},
{spans: span("a", "c\x00", "d", ""), key: "c", expFW: "c", expBW: "c"},
{spans: abc, key: "b", expFW: "b", expBW: "b"},
{spans: abc, key: "b\x00", expFW: "c", expBW: "b\x00"},
{spans: abc, key: "bb", expFW: "c", expBW: "b"},
{spans: span(), key: "whatevs", expFW: max, expBW: min},
{spans: span(loc("a"), loc("c")), key: "c", expFW: "c", expBW: "c"},
{spans: span(loc("a"), loc("c")), key: "c\x00", expFW: max, expBW: "c\x00"},
}
for i, test := range testCases {
var ba roachpb.BatchRequest
for _, span := range test.spans {
args := &roachpb.ScanRequest{}
args.Key, args.EndKey = span.Key, span.EndKey
ba.Add(args)
}
if next, err := next(ba, roachpb.RKey(test.key)); err != nil {
t.Errorf("%d: %v", i, err)
} else if !bytes.Equal(next, roachpb.Key(test.expFW)) {
t.Errorf("%d: next: expected %q, got %q", i, test.expFW, next)
}
if prev, err := prev(ba, roachpb.RKey(test.key)); err != nil {
t.Errorf("%d: %v", i, err)
} else if !bytes.Equal(prev, roachpb.Key(test.expBW)) {
t.Errorf("%d: prev: expected %q, got %q", i, test.expBW, prev)
}
}
}
// createRangeData creates sample range data in all possible areas of
// the key space. Returns a slice of the encoded keys of all created
// data.
func createRangeData(t *testing.T, r *Replica) []engine.MVCCKey {
ts0 := hlc.ZeroTimestamp
ts := hlc.Timestamp{WallTime: 1}
desc := r.Desc()
keyTSs := []struct {
key roachpb.Key
ts hlc.Timestamp
}{
{keys.AbortCacheKey(r.RangeID, testTxnID), ts0},
{keys.AbortCacheKey(r.RangeID, testTxnID2), ts0},
{keys.RangeFrozenStatusKey(r.RangeID), ts0},
{keys.RangeLastGCKey(r.RangeID), ts0},
{keys.RaftAppliedIndexKey(r.RangeID), ts0},
{keys.RaftTruncatedStateKey(r.RangeID), ts0},
{keys.RangeLeaseKey(r.RangeID), ts0},
{keys.LeaseAppliedIndexKey(r.RangeID), ts0},
{keys.RangeStatsKey(r.RangeID), ts0},
{keys.RangeTxnSpanGCThresholdKey(r.RangeID), ts0},
{keys.RaftHardStateKey(r.RangeID), ts0},
{keys.RaftLastIndexKey(r.RangeID), ts0},
{keys.RaftLogKey(r.RangeID, 1), ts0},
{keys.RaftLogKey(r.RangeID, 2), ts0},
{keys.RangeLastReplicaGCTimestampKey(r.RangeID), ts0},
{keys.RangeLastVerificationTimestampKeyDeprecated(r.RangeID), ts0},
{keys.RangeDescriptorKey(desc.StartKey), ts},
{keys.TransactionKey(roachpb.Key(desc.StartKey), uuid.MakeV4()), ts0},
{keys.TransactionKey(roachpb.Key(desc.StartKey.Next()), uuid.MakeV4()), ts0},
{keys.TransactionKey(fakePrevKey(desc.EndKey), uuid.MakeV4()), ts0},
// TODO(bdarnell): KeyMin.Next() results in a key in the reserved system-local space.
// Once we have resolved https://github.com/cockroachdb/cockroach/issues/437,
// replace this with something that reliably generates the first valid key in the range.
//{r.Desc().StartKey.Next(), ts},
// The following line is similar to StartKey.Next() but adds more to the key to
// avoid falling into the system-local space.
{append(append([]byte{}, desc.StartKey...), '\x02'), ts},
{fakePrevKey(r.Desc().EndKey), ts},
}
keys := []engine.MVCCKey{}
for _, keyTS := range keyTSs {
if err := engine.MVCCPut(context.Background(), r.store.Engine(), nil, keyTS.key, keyTS.ts, roachpb.MakeValueFromString("value"), nil); err != nil {
t.Fatal(err)
}
keys = append(keys, engine.MVCCKey{Key: keyTS.key, Timestamp: keyTS.ts})
}
return keys
}
func TestTruncate(t *testing.T) {
defer leaktest.AfterTest(t)()
loc := func(s string) string {
return string(keys.RangeDescriptorKey(roachpb.RKey(s)))
}
locPrefix := func(s string) string {
return string(keys.MakeRangeKeyPrefix(roachpb.RKey(s)))
}
testCases := []struct {
keys [][2]string
expKeys [][2]string
from, to string
desc [2]string // optional, defaults to {from,to}
err string
}{
{
// Keys inside of active range.
keys: [][2]string{{"a", "q"}, {"c"}, {"b, e"}, {"q"}},
expKeys: [][2]string{{"a", "q"}, {"c"}, {"b, e"}, {"q"}},
from: "a", to: "q\x00",
},
{
// Keys outside of active range.
keys: [][2]string{{"a"}, {"a", "b"}, {"q"}, {"q", "z"}},
expKeys: [][2]string{{}, {}, {}, {}},
from: "b", to: "q",
},
{
// Range-local keys inside of active range.
keys: [][2]string{{loc("b")}, {loc("c")}},
expKeys: [][2]string{{loc("b")}, {loc("c")}},
from: "b", to: "e",
},
{
// Range-local key outside of active range.
keys: [][2]string{{loc("a")}},
expKeys: [][2]string{{}},
from: "b", to: "e",
},
{
// Range-local range contained in active range.
keys: [][2]string{{loc("b"), loc("e") + "\x00"}},
expKeys: [][2]string{{loc("b"), loc("e") + "\x00"}},
from: "b", to: "e\x00",
},
{
// Range-local range not contained in active range.
keys: [][2]string{{loc("a"), loc("b")}},
expKeys: [][2]string{{}},
from: "c", to: "e",
},
{
// Range-local range not contained in active range.
keys: [][2]string{{loc("a"), locPrefix("b")}, {loc("e"), loc("f")}},
expKeys: [][2]string{{}, {}},
from: "b", to: "e",
},
{
// Range-local range partially contained in active range.
keys: [][2]string{{loc("a"), loc("b")}},
expKeys: [][2]string{{loc("a"), locPrefix("b")}},
from: "a", to: "b",
},
{
// Range-local range partially contained in active range.
keys: [][2]string{{loc("a"), loc("b")}},
expKeys: [][2]string{{locPrefix("b"), loc("b")}},
from: "b", to: "e",
},
{
// Range-local range contained in active range.
keys: [][2]string{{locPrefix("b"), loc("b")}},
expKeys: [][2]string{{locPrefix("b"), loc("b")}},
from: "b", to: "c",
},
{
// Mixed range-local vs global key range.
keys: [][2]string{{loc("c"), "d\x00"}},
from: "b", to: "e",
err: "local key mixed with global key",
},
{
// Key range touching and intersecting active range.
keys: [][2]string{{"a", "b"}, {"a", "c"}, {"p", "q"}, {"p", "r"}, {"a", "z"}},
expKeys: [][2]string{{}, {"b", "c"}, {"p", "q"}, {"p", "q"}, {"b", "q"}},
from: "b", to: "q",
},
// Active key range is intersection of descriptor and [from,to).
{
keys: [][2]string{{"c", "q"}},
expKeys: [][2]string{{"d", "p"}},
from: "a", to: "z",
desc: [2]string{"d", "p"},
},
{
keys: [][2]string{{"c", "q"}},
expKeys: [][2]string{{"d", "p"}},
from: "d", to: "p",
desc: [2]string{"a", "z"},
},
}
for i, test := range testCases {
goldenOriginal := roachpb.BatchRequest{}
for _, ks := range test.keys {
if len(ks[1]) > 0 {
u := uuid.MakeV4()
goldenOriginal.Add(&roachpb.ResolveIntentRangeRequest{
Span: roachpb.Span{Key: roachpb.Key(ks[0]), EndKey: roachpb.Key(ks[1])},
IntentTxn: enginepb.TxnMeta{ID: &u},
})
} else {
goldenOriginal.Add(&roachpb.GetRequest{
Span: roachpb.Span{Key: roachpb.Key(ks[0])},
})
}
}
original := roachpb.BatchRequest{Requests: make([]roachpb.RequestUnion, len(goldenOriginal.Requests))}
for i, request := range goldenOriginal.Requests {
original.Requests[i].SetValue(request.GetInner().ShallowCopy())
}
desc := &roachpb.RangeDescriptor{
StartKey: roachpb.RKey(test.desc[0]), EndKey: roachpb.RKey(test.desc[1]),
}
if len(desc.StartKey) == 0 {
desc.StartKey = roachpb.RKey(test.from)
}
if len(desc.EndKey) == 0 {
desc.EndKey = roachpb.RKey(test.to)
}
rs := roachpb.RSpan{Key: roachpb.RKey(test.from), EndKey: roachpb.RKey(test.to)}
rs, err := rs.Intersect(desc)
if err != nil {
t.Errorf("%d: intersection failure: %v", i, err)
continue
}
ba, num, err := truncate(original, rs)
if err != nil || test.err != "" {
if !testutils.IsError(err, test.err) {
t.Errorf("%d: %v (expected: %q)", i, err, test.err)
}
continue
}
var reqs int
for j, arg := range ba.Requests {
req := arg.GetInner()
if _, ok := req.(*roachpb.NoopRequest); ok {
continue
}
if h := req.Header(); !bytes.Equal(h.Key, roachpb.Key(test.expKeys[j][0])) || !bytes.Equal(h.EndKey, roachpb.Key(test.expKeys[j][1])) {
t.Errorf("%d.%d: range mismatch: actual [%q,%q), wanted [%q,%q)", i, j,
h.Key, h.EndKey, test.expKeys[j][0], test.expKeys[j][1])
} else if _, ok := req.(*roachpb.NoopRequest); ok != (len(h.Key) == 0) {
t.Errorf("%d.%d: expected NoopRequest, got %T", i, j, req)
} else if len(h.Key) != 0 {
reqs++
}
}
if reqs != num {
t.Errorf("%d: counted %d requests, but truncation indicated %d", i, reqs, num)
}
if !reflect.DeepEqual(original, goldenOriginal) {
t.Errorf("%d: truncation mutated original:\nexpected: %s\nactual: %s",
i, goldenOriginal, original)
}
}
}
// TestStoreRangeMergeWithData attempts to merge two collocate ranges
// each containing data.
func TestStoreRangeMergeWithData(t *testing.T) {
defer leaktest.AfterTest(t)()
storeCfg := storage.TestStoreConfig(nil)
storeCfg.TestingKnobs.DisableSplitQueue = true
store, stopper := createTestStoreWithConfig(t, storeCfg)
defer stopper.Stop()
content := roachpb.Key("testing!")
aDesc, bDesc, err := createSplitRanges(store)
if err != nil {
t.Fatal(err)
}
// Write some values left and right of the proposed split key.
pArgs := putArgs([]byte("aaa"), content)
if _, err := client.SendWrapped(context.Background(), rg1(store), &pArgs); err != nil {
t.Fatal(err)
}
pArgs = putArgs([]byte("ccc"), content)
if _, err := client.SendWrappedWith(context.Background(), rg1(store), roachpb.Header{
RangeID: bDesc.RangeID,
}, &pArgs); err != nil {
t.Fatal(err)
}
// Confirm the values are there.
gArgs := getArgs([]byte("aaa"))
if reply, err := client.SendWrapped(context.Background(), rg1(store), &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("ccc"))
if reply, err := client.SendWrappedWith(context.Background(), rg1(store), roachpb.Header{
RangeID: bDesc.RangeID,
}, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
// Merge the b range back into the a range.
args := adminMergeArgs(roachpb.KeyMin)
if _, err := client.SendWrapped(context.Background(), rg1(store), &args); err != nil {
t.Fatal(err)
}
// Verify no intents remains on range descriptor keys.
for _, key := range []roachpb.Key{keys.RangeDescriptorKey(aDesc.StartKey), keys.RangeDescriptorKey(bDesc.StartKey)} {
if _, _, err := engine.MVCCGet(context.Background(), store.Engine(), key, store.Clock().Now(), true, nil); err != nil {
t.Fatal(err)
}
}
// Verify the merge by looking up keys from both ranges.
rangeA := store.LookupReplica([]byte("a"), nil)
rangeB := store.LookupReplica([]byte("c"), nil)
rangeADesc := rangeA.Desc()
rangeBDesc := rangeB.Desc()
if !reflect.DeepEqual(rangeA, rangeB) {
t.Fatalf("ranges were not merged %+v=%+v", rangeADesc, rangeBDesc)
}
if !bytes.Equal(rangeADesc.StartKey, roachpb.RKeyMin) {
t.Fatalf("The start key is not equal to KeyMin %q=%q", rangeADesc.StartKey, roachpb.RKeyMin)
}
if !bytes.Equal(rangeADesc.EndKey, roachpb.RKeyMax) {
t.Fatalf("The end key is not equal to KeyMax %q=%q", rangeADesc.EndKey, roachpb.RKeyMax)
}
// Try to get values from after the merge.
gArgs = getArgs([]byte("aaa"))
if reply, err := client.SendWrapped(context.Background(), rg1(store), &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("ccc"))
if reply, err := client.SendWrappedWith(context.Background(), rg1(store), roachpb.Header{
RangeID: rangeB.RangeID,
}, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
// Put new values after the merge on both sides.
pArgs = putArgs([]byte("aaaa"), content)
if _, err := client.SendWrapped(context.Background(), rg1(store), &pArgs); err != nil {
t.Fatal(err)
}
pArgs = putArgs([]byte("cccc"), content)
if _, err := client.SendWrappedWith(context.Background(), rg1(store), roachpb.Header{
RangeID: rangeB.RangeID,
}, &pArgs); err != nil {
t.Fatal(err)
}
// Try to get the newly placed values.
gArgs = getArgs([]byte("aaaa"))
if reply, err := client.SendWrapped(context.Background(), rg1(store), &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("cccc"))
if reply, err := client.SendWrapped(context.Background(), rg1(store), &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
}
func runDebugGCCmd(cmd *cobra.Command, args []string) error {
stopper := stop.NewStopper()
defer stopper.Stop()
if len(args) != 1 {
return errors.New("one argument required: dir")
}
var rangeID roachpb.RangeID
if len(args) == 2 {
var err error
if rangeID, err = parseRangeID(args[1]); err != nil {
return err
}
}
db, err := openStore(cmd, args[0], stopper)
if err != nil {
return err
}
start := keys.RangeDescriptorKey(roachpb.RKeyMin)
end := keys.RangeDescriptorKey(roachpb.RKeyMax)
var descs []roachpb.RangeDescriptor
if _, err := engine.MVCCIterate(context.Background(), db, start, end, hlc.MaxTimestamp,
false /* !consistent */, nil, /* txn */
false /* !reverse */, func(kv roachpb.KeyValue) (bool, error) {
var desc roachpb.RangeDescriptor
_, suffix, _, err := keys.DecodeRangeKey(kv.Key)
if err != nil {
return false, err
}
if !bytes.Equal(suffix, keys.LocalRangeDescriptorSuffix) {
return false, nil
}
if err := kv.Value.GetProto(&desc); err != nil {
return false, err
}
if desc.RangeID == rangeID || rangeID == 0 {
descs = append(descs, desc)
}
return desc.RangeID == rangeID, nil
}); err != nil {
return err
}
if len(descs) == 0 {
return fmt.Errorf("no range matching the criteria found")
}
for _, desc := range descs {
snap := db.NewSnapshot()
defer snap.Close()
_, info, err := storage.RunGC(context.Background(), &desc, snap, hlc.Timestamp{WallTime: timeutil.Now().UnixNano()},
config.GCPolicy{TTLSeconds: 24 * 60 * 60 /* 1 day */}, func(_ hlc.Timestamp, _ *roachpb.Transaction, _ roachpb.PushTxnType) {
}, func(_ []roachpb.Intent, _, _ bool) error { return nil })
if err != nil {
return err
}
fmt.Printf("RangeID: %d [%s, %s):\n", desc.RangeID, desc.StartKey, desc.EndKey)
_, _ = pretty.Println(info)
}
return nil
}
// StartCluster starts a cluster from the relevant flags. All test clusters
// should be created through this command since it sets up the logging in a
// unified way.
func StartCluster(t *testing.T, cfg cluster.TestConfig) (c cluster.Cluster) {
var completed bool
defer func() {
if !completed && c != nil {
c.AssertAndStop(t)
}
}()
if *flagRemote {
f := farmer(t, "")
c = f
if err := f.Resize(*flagNodes); err != nil {
t.Fatal(err)
}
if err := f.WaitReady(5 * time.Minute); err != nil {
if destroyErr := f.Destroy(t); destroyErr != nil {
t.Fatalf("could not destroy cluster after error %s: %s", err, destroyErr)
}
t.Fatalf("cluster not ready in time: %s", err)
}
} else {
logDir := *flagLogDir
if logDir != "" {
logDir = func(d string) string {
for i := 1; i < 100; i++ {
_, _, fun := caller.Lookup(i)
if testFuncRE.MatchString(fun) {
return filepath.Join(d, fun)
}
}
panic("no caller matching Test(.*) in stack trace")
}(logDir)
}
l := cluster.CreateLocal(cfg, logDir, *flagPrivileged, stopper)
l.Start()
c = l
}
wantedReplicas := 3
if numNodes := c.NumNodes(); numNodes < wantedReplicas {
wantedReplicas = numNodes
}
// Looks silly, but we actually start zero-node clusters in the
// reference tests.
if wantedReplicas > 0 {
ctx := context.TODO()
log.Infof(ctx, "waiting for first range to have %d replicas", wantedReplicas)
util.SucceedsSoon(t, func() error {
select {
case <-stopper:
t.Fatal("interrupted")
case <-time.After(time.Second):
}
// Reconnect on every iteration; gRPC will eagerly tank the connection
// on transport errors. Always talk to node 0 because it's guaranteed
// to exist.
client, dbStopper := c.NewClient(t, 0)
defer dbStopper.Stop()
ctxWithTimeout, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
var desc roachpb.RangeDescriptor
if err := client.GetProto(ctxWithTimeout, keys.RangeDescriptorKey(roachpb.RKeyMin), &desc); err != nil {
return err
}
foundReplicas := len(desc.Replicas)
if log.V(1) {
log.Infof(ctxWithTimeout, "found %d replicas", foundReplicas)
}
if foundReplicas < wantedReplicas {
return errors.Errorf("expected %d replicas, only found %d", wantedReplicas, foundReplicas)
}
return nil
})
}
completed = true
return c
}
func TestLogRebalances(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, db := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
// Use a client to get the RangeDescriptor for the first range. We will use
// this range's information to log fake rebalance events.
desc := &roachpb.RangeDescriptor{}
if err := db.GetProto(context.TODO(), keys.RangeDescriptorKey(roachpb.RKeyMin), desc); err != nil {
t.Fatal(err)
}
// This code assumes that there is only one TestServer, and thus that
// StoreID 1 is present on the testserver. If this assumption changes in the
// future, *any* store will work, but a new method will need to be added to
// Stores (or a creative usage of VisitStores could suffice).
store, err := s.(*server.TestServer).Stores().GetStore(roachpb.StoreID(1))
if err != nil {
t.Fatal(err)
}
// Log several fake events using the store.
logEvent := func(changeType roachpb.ReplicaChangeType) {
if err := db.Txn(context.TODO(), func(txn *client.Txn) error {
return store.LogReplicaChangeTest(txn, changeType, desc.Replicas[0], *desc)
}); err != nil {
t.Fatal(err)
}
}
checkMetrics := func(expAdds, expRemoves int64) {
if a, e := store.Metrics().RangeAdds.Count(), expAdds; a != e {
t.Errorf("range adds %d != expected %d", a, e)
}
if a, e := store.Metrics().RangeRemoves.Count(), expRemoves; a != e {
t.Errorf("range removes %d != expected %d", a, e)
}
}
logEvent(roachpb.ADD_REPLICA)
checkMetrics(1 /*add*/, 0 /*remove*/)
logEvent(roachpb.ADD_REPLICA)
checkMetrics(2 /*adds*/, 0 /*remove*/)
logEvent(roachpb.REMOVE_REPLICA)
checkMetrics(2 /*adds*/, 1 /*remove*/)
// Open a SQL connection to verify that the events have been logged.
pgURL, cleanupFn := sqlutils.PGUrl(t, s.ServingAddr(), "TestLogRebalances", url.User(security.RootUser))
defer cleanupFn()
sqlDB, err := gosql.Open("postgres", pgURL.String())
if err != nil {
t.Fatal(err)
}
defer sqlDB.Close()
// verify that two add replica events have been logged.
// TODO(mrtracy): placeholders still appear to be broken, this query should
// be using a string placeholder for the eventType value.
rows, err := sqlDB.Query(`SELECT rangeID, info FROM system.rangelog WHERE eventType = 'add'`)
if err != nil {
t.Fatal(err)
}
var count int
for rows.Next() {
count++
var rangeID int64
var infoStr gosql.NullString
if err := rows.Scan(&rangeID, &infoStr); err != nil {
t.Fatal(err)
}
if a, e := roachpb.RangeID(rangeID), desc.RangeID; a != e {
t.Errorf("wrong rangeID %d recorded for add event, expected %d", a, e)
}
// Verify that info returns a json struct.
if !infoStr.Valid {
t.Errorf("info not recorded for add replica of range %d", rangeID)
}
var info struct {
AddReplica roachpb.ReplicaDescriptor
UpdatedDesc roachpb.RangeDescriptor
}
if err := json.Unmarshal([]byte(infoStr.String), &info); err != nil {
t.Errorf("error unmarshalling info string for add replica %d: %s", rangeID, err)
continue
}
if int64(info.UpdatedDesc.RangeID) != rangeID {
t.Errorf("recorded wrong updated descriptor %s for add replica of range %d", info.UpdatedDesc, rangeID)
}
if a, e := info.AddReplica, desc.Replicas[0]; a != e {
t.Errorf("recorded wrong updated replica %s for add replica of range %d, expected %s",
a, rangeID, e)
}
}
if rows.Err() != nil {
t.Fatal(rows.Err())
}
if a, e := count, 2; a != e {
t.Errorf("expected %d AddReplica events logged, found %d", e, a)
}
// verify that one remove replica event was logged.
rows, err = sqlDB.Query(`SELECT rangeID, info FROM system.rangelog WHERE eventType = 'remove'`)
if err != nil {
t.Fatal(err)
}
count = 0
for rows.Next() {
count++
var rangeID int64
var infoStr gosql.NullString
if err := rows.Scan(&rangeID, &infoStr); err != nil {
t.Fatal(err)
}
if a, e := roachpb.RangeID(rangeID), desc.RangeID; a != e {
t.Errorf("wrong rangeID %d recorded for remove event, expected %d", a, e)
}
// Verify that info returns a json struct.
if !infoStr.Valid {
t.Errorf("info not recorded for remove replica of range %d", rangeID)
}
var info struct {
RemovedReplica roachpb.ReplicaDescriptor
UpdatedDesc roachpb.RangeDescriptor
}
if err := json.Unmarshal([]byte(infoStr.String), &info); err != nil {
t.Errorf("error unmarshalling info string for remove replica %d: %s", rangeID, err)
continue
}
if int64(info.UpdatedDesc.RangeID) != rangeID {
t.Errorf("recorded wrong updated descriptor %s for remove replica of range %d", info.UpdatedDesc, rangeID)
}
if a, e := info.RemovedReplica, desc.Replicas[0]; a != e {
t.Errorf("recorded wrong updated replica %s for remove replica of range %d, expected %s",
a, rangeID, e)
}
}
if rows.Err() != nil {
t.Fatal(rows.Err())
}
if a, e := count, 1; a != e {
t.Errorf("expected %d RemoveReplica events logged, found %d", e, a)
}
}
// snapshot creates an OutgoingSnapshot containing a rocksdb snapshot for the given range.
func snapshot(
ctx context.Context,
snapType string,
snap engine.Reader,
rangeID roachpb.RangeID,
eCache *raftEntryCache,
startKey roachpb.RKey,
) (OutgoingSnapshot, error) {
var desc roachpb.RangeDescriptor
// We ignore intents on the range descriptor (consistent=false) because we
// know they cannot be committed yet; operations that modify range
// descriptors resolve their own intents when they commit.
ok, err := engine.MVCCGetProto(ctx, snap, keys.RangeDescriptorKey(startKey),
hlc.MaxTimestamp, false /* !consistent */, nil, &desc)
if err != nil {
return OutgoingSnapshot{}, errors.Errorf("failed to get desc: %s", err)
}
if !ok {
return OutgoingSnapshot{}, errors.Errorf("couldn't find range descriptor")
}
var snapData roachpb.RaftSnapshotData
// Store RangeDescriptor as metadata, it will be retrieved by ApplySnapshot()
snapData.RangeDescriptor = desc
// Read the range metadata from the snapshot instead of the members
// of the Range struct because they might be changed concurrently.
appliedIndex, _, err := loadAppliedIndex(ctx, snap, rangeID)
if err != nil {
return OutgoingSnapshot{}, err
}
// Synthesize our raftpb.ConfState from desc.
var cs raftpb.ConfState
for _, rep := range desc.Replicas {
cs.Nodes = append(cs.Nodes, uint64(rep.ReplicaID))
}
term, err := term(ctx, snap, rangeID, eCache, appliedIndex)
if err != nil {
return OutgoingSnapshot{}, errors.Errorf("failed to fetch term of %d: %s", appliedIndex, err)
}
state, err := loadState(ctx, snap, &desc)
if err != nil {
return OutgoingSnapshot{}, err
}
// Intentionally let this iterator and the snapshot escape so that the
// streamer can send chunks from it bit by bit.
iter := NewReplicaDataIterator(&desc, snap, true /* replicatedOnly */)
snapUUID := uuid.MakeV4()
log.Infof(ctx, "generated %s snapshot %s at index %d",
snapType, snapUUID.Short(), appliedIndex)
return OutgoingSnapshot{
EngineSnap: snap,
Iter: iter,
State: state,
SnapUUID: snapUUID,
RaftSnap: raftpb.Snapshot{
Data: snapUUID.GetBytes(),
Metadata: raftpb.SnapshotMetadata{
Index: appliedIndex,
Term: term,
ConfState: cs,
},
},
}, nil
}
// StartCluster starts a cluster from the relevant flags. All test clusters
// should be created through this command since it sets up the logging in a
// unified way.
func StartCluster(ctx context.Context, t *testing.T, cfg cluster.TestConfig) (c cluster.Cluster) {
var completed bool
defer func() {
if !completed && c != nil {
c.AssertAndStop(ctx, t)
}
}()
if *flagRemote {
f := MakeFarmer(t, "", stopper)
c = f
if err := f.Resize(*flagNodes); err != nil {
t.Fatal(err)
}
if err := f.WaitReady(5 * time.Minute); err != nil {
if destroyErr := f.Destroy(t); destroyErr != nil {
t.Fatalf("could not destroy cluster after error %s: %s", err, destroyErr)
}
t.Fatalf("cluster not ready in time: %s", err)
}
} else {
logDir := *flagLogDir
if logDir != "" {
logDir = func(d string) string {
for i := 1; i < 100; i++ {
_, _, fun := caller.Lookup(i)
if testFuncRE.MatchString(fun) {
return filepath.Join(d, fun)
}
}
panic("no caller matching Test(.*) in stack trace")
}(logDir)
}
l := cluster.CreateLocal(ctx, cfg, logDir, *flagPrivileged, stopper)
l.Start(ctx)
c = l
}
wantedReplicas := 3
if numNodes := c.NumNodes(); numNodes < wantedReplicas {
wantedReplicas = numNodes
}
// Looks silly, but we actually start zero-node clusters in the
// reference tests.
if wantedReplicas > 0 {
log.Infof(ctx, "waiting for first range to have %d replicas", wantedReplicas)
testutils.SucceedsSoon(t, func() error {
select {
case <-stopper.ShouldStop():
t.Fatal("interrupted")
case <-time.After(time.Second):
}
// Reconnect on every iteration; gRPC will eagerly tank the connection
// on transport errors. Always talk to node 0 because it's guaranteed
// to exist.
client, err := c.NewClient(ctx, 0)
if err != nil {
t.Fatal(err)
}
var desc roachpb.RangeDescriptor
if err := client.GetProto(ctx, keys.RangeDescriptorKey(roachpb.RKeyMin), &desc); err != nil {
return err
}
foundReplicas := len(desc.Replicas)
if log.V(1) {
log.Infof(ctx, "found %d replicas", foundReplicas)
}
if foundReplicas < wantedReplicas {
return errors.Errorf("expected %d replicas, only found %d", wantedReplicas, foundReplicas)
}
return nil
})
}
// Ensure that all nodes are serving SQL by making sure a simple
// read-only query succeeds.
for i := 0; i < c.NumNodes(); i++ {
testutils.SucceedsSoon(t, func() error {
db, err := gosql.Open("postgres", c.PGUrl(ctx, i))
if err != nil {
return err
}
if _, err := db.Exec("SHOW DATABASES;"); err != nil {
return err
}
return nil
})
}
completed = true
return c
}