// TestMultiRangeScanWithMaxResults tests that commands which access multiple // ranges with MaxResults parameter are carried out properly. func TestMultiRangeScanWithMaxResults(t *testing.T) { defer leaktest.AfterTest(t)() testCases := []struct { splitKeys []roachpb.Key keys []roachpb.Key }{ {[]roachpb.Key{roachpb.Key("m")}, []roachpb.Key{roachpb.Key("a"), roachpb.Key("z")}}, {[]roachpb.Key{roachpb.Key("h"), roachpb.Key("q")}, []roachpb.Key{roachpb.Key("b"), roachpb.Key("f"), roachpb.Key("k"), roachpb.Key("r"), roachpb.Key("w"), roachpb.Key("y")}}, } for i, tc := range testCases { s, _, _ := serverutils.StartServer(t, base.TestServerArgs{}) defer s.Stopper().Stop() ts := s.(*TestServer) retryOpts := base.DefaultRetryOptions() retryOpts.Closer = ts.stopper.ShouldDrain() ds := kv.NewDistSender(&kv.DistSenderContext{ Clock: s.Clock(), RPCContext: s.RPCContext(), RPCRetryOptions: &retryOpts, }, ts.Gossip()) tds := kv.NewTxnCoordSender(ds, ts.Clock(), ts.Ctx.Linearizable, tracing.NewTracer(), ts.stopper, kv.NewTxnMetrics(metric.NewRegistry())) for _, sk := range tc.splitKeys { if err := ts.node.ctx.DB.AdminSplit(sk); err != nil { t.Fatal(err) } } for _, k := range tc.keys { put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k)) if _, err := client.SendWrapped(tds, nil, put); err != nil { t.Fatal(err) } } // Try every possible ScanRequest startKey. for start := 0; start < len(tc.keys); start++ { // Try every possible maxResults, from 1 to beyond the size of key array. for maxResults := 1; maxResults <= len(tc.keys)-start+1; maxResults++ { scan := roachpb.NewScan(tc.keys[start], tc.keys[len(tc.keys)-1].Next(), int64(maxResults)) reply, err := client.SendWrapped(tds, nil, scan) if err != nil { t.Fatal(err) } rows := reply.(*roachpb.ScanResponse).Rows if start+maxResults <= len(tc.keys) && len(rows) != maxResults { t.Errorf("%d: start=%s: expected %d rows, but got %d", i, tc.keys[start], maxResults, len(rows)) } else if start+maxResults == len(tc.keys)+1 && len(rows) != maxResults-1 { t.Errorf("%d: expected %d rows, but got %d", i, maxResults-1, len(rows)) } } } } }
// TestMultiRangeScanReverseScanInconsistent verifies that a Scan/ReverseScan // across ranges that doesn't require read consistency will set a timestamp // using the clock local to the distributed sender. func TestMultiRangeScanReverseScanInconsistent(t *testing.T) { defer leaktest.AfterTest(t) s, db := setupMultipleRanges(t, "b") defer s.Stop() // Write keys "a" and "b", the latter of which is the first key in the // second range. keys := []string{"a", "b"} ts := []time.Time{} for i, key := range keys { b := &client.Batch{} b.Put(key, "value") if err := db.Run(b); err != nil { t.Fatal(err) } ts = append(ts, b.Results[0].Rows[0].Timestamp()) log.Infof("%d: %s", i, b.Results[0].Rows[0].Timestamp()) } // Do an inconsistent Scan/ReverseScan from a new DistSender and verify // it does the read at its local clock and doesn't receive an // OpRequiresTxnError. We set the local clock to the timestamp of // the first key to verify it's used to read only key "a". manual := hlc.NewManualClock(ts[1].UnixNano() - 1) clock := hlc.NewClock(manual.UnixNano) ds := kv.NewDistSender(&kv.DistSenderContext{Clock: clock}, s.Gossip()) // Scan. sa := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("c"), 0).(*roachpb.ScanRequest) reply, err := client.SendWrappedWith(ds, nil, roachpb.BatchRequest_Header{ ReadConsistency: roachpb.INCONSISTENT, }, sa) if err != nil { t.Fatal(err) } sr := reply.(*roachpb.ScanResponse) if l := len(sr.Rows); l != 1 { t.Fatalf("expected 1 row; got %d", l) } if key := string(sr.Rows[0].Key); keys[0] != key { t.Errorf("expected key %q; got %q", keys[0], key) } // ReverseScan. rsa := roachpb.NewReverseScan(roachpb.Key("a"), roachpb.Key("c"), 0).(*roachpb.ReverseScanRequest) reply, err = client.SendWrappedWith(ds, nil, roachpb.BatchRequest_Header{ ReadConsistency: roachpb.INCONSISTENT, }, rsa) if err != nil { t.Fatal(err) } rsr := reply.(*roachpb.ReverseScanResponse) if l := len(rsr.Rows); l != 1 { t.Fatalf("expected 1 row; got %d", l) } if key := string(rsr.Rows[0].Key); keys[0] != key { t.Errorf("expected key %q; got %q", keys[0], key) } }
// TestSendRPCRetry verifies that sendRPC failed on first address but succeed on // second address, the second reply should be successfully returned back. func TestSendRPCRetry(t *testing.T) { defer leaktest.AfterTest(t) g, s := makeTestGossip(t) defer s() g.SetNodeID(1) if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil { t.Fatal(err) } // Fill RangeDescriptor with 2 replicas var descriptor = roachpb.RangeDescriptor{ RangeID: 1, StartKey: roachpb.RKey("a"), EndKey: roachpb.RKey("z"), } for i := 1; i <= 2; i++ { addr := util.MakeUnresolvedAddr("tcp", fmt.Sprintf("node%d", i)) nd := &roachpb.NodeDescriptor{ NodeID: roachpb.NodeID(i), Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()), } if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(i)), nd, time.Hour); err != nil { t.Fatal(err) } descriptor.Replicas = append(descriptor.Replicas, roachpb.ReplicaDescriptor{ NodeID: roachpb.NodeID(i), StoreID: roachpb.StoreID(i), }) } // Define our rpcSend stub which returns success on the second address. var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) { if method == "Node.Batch" { // reply from first address failed _ = getReply() // reply from second address succeed batchReply := getReply().(*roachpb.BatchResponse) reply := &roachpb.ScanResponse{} batchReply.Add(reply) reply.Rows = append([]roachpb.KeyValue{}, roachpb.KeyValue{Key: roachpb.Key("b"), Value: roachpb.Value{}}) return []proto.Message{batchReply}, nil } return nil, util.Errorf("unexpected method %v", method) } ctx := &DistSenderContext{ RPCSend: testFn, RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) { return []roachpb.RangeDescriptor{descriptor}, nil }), } ds := NewDistSender(ctx, g) scan := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("d"), 1) sr, err := client.SendWrapped(ds, nil, scan) if err != nil { t.Fatal(err) } if l := len(sr.(*roachpb.ScanResponse).Rows); l != 1 { t.Fatalf("expected 1 row; got %d", l) } }
// TestRetryOnWrongReplicaError sets up a DistSender on a minimal gossip // network and a mock of rpc.Send, and verifies that the DistSender correctly // retries upon encountering a stale entry in its range descriptor cache. func TestRetryOnWrongReplicaError(t *testing.T) { defer leaktest.AfterTest(t) g, s := makeTestGossip(t) defer s() // Updated below, after it has first been returned. badStartKey := roachpb.RKey("m") newRangeDescriptor := testRangeDescriptor goodStartKey := newRangeDescriptor.StartKey newRangeDescriptor.StartKey = badStartKey descStale := true var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) { ba := getArgs(testAddress).(*roachpb.BatchRequest) rs := keys.Range(*ba) if _, ok := ba.GetArg(roachpb.RangeLookup); ok { if !descStale && bytes.HasPrefix(rs.Key, keys.Meta2Prefix) { t.Errorf("unexpected extra lookup for non-stale replica descriptor at %s", rs.Key) } br := getReply().(*roachpb.BatchResponse) r := &roachpb.RangeLookupResponse{} r.Ranges = append(r.Ranges, newRangeDescriptor) br.Add(r) // If we just returned the stale descriptor, set up returning the // good one next time. if bytes.HasPrefix(rs.Key, keys.Meta2Prefix) { if newRangeDescriptor.StartKey.Equal(badStartKey) { newRangeDescriptor.StartKey = goodStartKey } else { descStale = false } } return []proto.Message{br}, nil } // When the Scan first turns up, update the descriptor for future // range descriptor lookups. if !newRangeDescriptor.StartKey.Equal(goodStartKey) { return nil, &roachpb.RangeKeyMismatchError{RequestStartKey: rs.Key.AsRawKey(), RequestEndKey: rs.EndKey.AsRawKey()} } return []proto.Message{ba.CreateReply()}, nil } ctx := &DistSenderContext{ RPCSend: testFn, } ds := NewDistSender(ctx, g) scan := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("d"), 0) if _, err := client.SendWrapped(ds, nil, scan); err != nil { t.Errorf("scan encountered error: %s", err) } }
// TestSendRPCRetry verifies that sendRPC failed on first address but succeed on // second address, the second reply should be successfully returned back. func TestSendRPCRetry(t *testing.T) { defer leaktest.AfterTest(t) g, s := makeTestGossip(t) defer s() g.SetNodeID(1) if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil { t.Fatal(err) } // Fill RangeDescriptor with 2 replicas var descriptor = roachpb.RangeDescriptor{ RangeID: 1, StartKey: roachpb.RKey("a"), EndKey: roachpb.RKey("z"), } for i := 1; i <= 2; i++ { addr := util.MakeUnresolvedAddr("tcp", fmt.Sprintf("node%d", i)) nd := &roachpb.NodeDescriptor{ NodeID: roachpb.NodeID(i), Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()), } if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(i)), nd, time.Hour); err != nil { t.Fatal(err) } descriptor.Replicas = append(descriptor.Replicas, roachpb.ReplicaDescriptor{ NodeID: roachpb.NodeID(i), StoreID: roachpb.StoreID(i), }) } var testFn rpcSendFn = func(_ SendOptions, _ ReplicaSlice, args roachpb.BatchRequest, _ *rpc.Context) (proto.Message, error) { batchReply := &roachpb.BatchResponse{} reply := &roachpb.ScanResponse{} batchReply.Add(reply) reply.Rows = append([]roachpb.KeyValue{}, roachpb.KeyValue{Key: roachpb.Key("b"), Value: roachpb.Value{}}) return batchReply, nil } ctx := &DistSenderContext{ RPCSend: testFn, RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) { return []roachpb.RangeDescriptor{descriptor}, nil }), } ds := NewDistSender(ctx, g) scan := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("d"), 1) sr, err := client.SendWrapped(ds, nil, scan) if err != nil { t.Fatal(err) } if l := len(sr.(*roachpb.ScanResponse).Rows); l != 1 { t.Fatalf("expected 1 row; got %d", l) } }
// TestMultiRangeScanWithMaxResults tests that commands which access multiple // ranges with MaxResults parameter are carried out properly. func TestMultiRangeScanWithMaxResults(t *testing.T) { defer leaktest.AfterTest(t) testCases := []struct { splitKeys []roachpb.Key keys []roachpb.Key }{ {[]roachpb.Key{roachpb.Key("m")}, []roachpb.Key{roachpb.Key("a"), roachpb.Key("z")}}, {[]roachpb.Key{roachpb.Key("h"), roachpb.Key("q")}, []roachpb.Key{roachpb.Key("b"), roachpb.Key("f"), roachpb.Key("k"), roachpb.Key("r"), roachpb.Key("w"), roachpb.Key("y")}}, } for i, tc := range testCases { s := StartTestServer(t) ds := kv.NewDistSender(&kv.DistSenderContext{Clock: s.Clock()}, s.Gossip()) tds := kv.NewTxnCoordSender(ds, s.Clock(), testContext.Linearizable, nil, s.stopper) for _, sk := range tc.splitKeys { if err := s.node.ctx.DB.AdminSplit(sk); err != nil { t.Fatal(err) } } for _, k := range tc.keys { put := roachpb.NewPut(k, roachpb.Value{Bytes: k}) if _, err := client.SendWrapped(tds, nil, put); err != nil { t.Fatal(err) } } // Try every possible ScanRequest startKey. for start := 0; start < len(tc.keys); start++ { // Try every possible maxResults, from 1 to beyond the size of key array. for maxResults := 1; maxResults <= len(tc.keys)-start+1; maxResults++ { scan := roachpb.NewScan(tc.keys[start], tc.keys[len(tc.keys)-1].Next(), int64(maxResults)) reply, err := client.SendWrapped(tds, nil, scan) if err != nil { t.Fatal(err) } rows := reply.(*roachpb.ScanResponse).Rows if start+maxResults <= len(tc.keys) && len(rows) != maxResults { t.Errorf("%d: start=%s: expected %d rows, but got %d", i, tc.keys[start], maxResults, len(rows)) } else if start+maxResults == len(tc.keys)+1 && len(rows) != maxResults-1 { t.Errorf("%d: expected %d rows, but got %d", i, maxResults-1, len(rows)) } } } defer s.Stop() } }
func (b *Batch) scan(s, e interface{}, maxRows int64, isReverse bool) { begin, err := marshalKey(s) if err != nil { b.initResult(0, 0, err) return } end, err := marshalKey(e) if err != nil { b.initResult(0, 0, err) return } if !isReverse { b.reqs = append(b.reqs, roachpb.NewScan(roachpb.Key(begin), roachpb.Key(end), maxRows)) } else { b.reqs = append(b.reqs, roachpb.NewReverseScan(roachpb.Key(begin), roachpb.Key(end), maxRows)) } b.initResult(1, 0, nil) }
func (b *Batch) scan(s, e interface{}, isReverse bool) { begin, err := marshalKey(s) if err != nil { b.initResult(0, 0, notRaw, err) return } end, err := marshalKey(e) if err != nil { b.initResult(0, 0, notRaw, err) return } if !isReverse { b.appendReqs(roachpb.NewScan(begin, end)) } else { b.appendReqs(roachpb.NewReverseScan(begin, end)) } b.initResult(1, 0, notRaw, nil) }
// TestMultiRangeMergeStaleDescriptor simulates the situation in which the // DistSender executes a multi-range scan which encounters the stale descriptor // of a range which has since incorporated its right neighbor by means of a // merge. It is verified that the DistSender scans the correct keyrange exactly // once. func TestMultiRangeMergeStaleDescriptor(t *testing.T) { defer leaktest.AfterTest(t) g, s := makeTestGossip(t) defer s() // Assume we have two ranges, [a-b) and [b-KeyMax). merged := false // The stale first range descriptor which is unaware of the merge. var FirstRange = roachpb.RangeDescriptor{ RangeID: 1, StartKey: roachpb.RKey("a"), EndKey: roachpb.RKey("b"), Replicas: []roachpb.ReplicaDescriptor{ { NodeID: 1, StoreID: 1, }, }, } // The merged descriptor, which will be looked up after having processed // the stale range [a,b). var mergedRange = roachpb.RangeDescriptor{ RangeID: 1, StartKey: roachpb.RKey("a"), EndKey: roachpb.RKeyMax, Replicas: []roachpb.ReplicaDescriptor{ { NodeID: 1, StoreID: 1, }, }, } // Assume we have two key-value pairs, a=1 and c=2. existingKVs := []roachpb.KeyValue{ {Key: roachpb.Key("a"), Value: roachpb.MakeValueFromString("1")}, {Key: roachpb.Key("c"), Value: roachpb.MakeValueFromString("2")}, } var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) { if method != "Node.Batch" { t.Fatalf("unexpected method:%s", method) } ba := getArgs(testAddress).(*roachpb.BatchRequest) rs := keys.Range(*ba) batchReply := getReply().(*roachpb.BatchResponse) reply := &roachpb.ScanResponse{} batchReply.Add(reply) results := []roachpb.KeyValue{} for _, curKV := range existingKVs { if rs.Key.Less(keys.Addr(curKV.Key).Next()) && keys.Addr(curKV.Key).Less(rs.EndKey) { results = append(results, curKV) } } reply.Rows = results return []proto.Message{batchReply}, nil } ctx := &DistSenderContext{ RPCSend: testFn, RangeDescriptorDB: mockRangeDescriptorDB(func(key roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) { if !merged { // Assume a range merge operation happened. merged = true return []roachpb.RangeDescriptor{FirstRange}, nil } return []roachpb.RangeDescriptor{mergedRange}, nil }), } ds := NewDistSender(ctx, g) scan := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("d"), 10).(*roachpb.ScanRequest) // Set the Txn info to avoid an OpRequiresTxnError. reply, err := client.SendWrappedWith(ds, nil, roachpb.Header{ Txn: &roachpb.Transaction{}, }, scan) if err != nil { t.Fatalf("scan encountered error: %s", err) } sr := reply.(*roachpb.ScanResponse) if !reflect.DeepEqual(existingKVs, sr.Rows) { t.Fatalf("expect get %v, actual get %v", existingKVs, sr.Rows) } }
// TestMultiRangeScanDeleteRange tests that commands which access multiple // ranges are carried out properly. func TestMultiRangeScanDeleteRange(t *testing.T) { defer leaktest.AfterTest(t) s := StartTestServer(t) defer s.Stop() ds := kv.NewDistSender(&kv.DistSenderContext{Clock: s.Clock()}, s.Gossip()) tds := kv.NewTxnCoordSender(ds, s.Clock(), testContext.Linearizable, nil, s.stopper) if err := s.node.ctx.DB.AdminSplit("m"); err != nil { t.Fatal(err) } writes := []roachpb.Key{roachpb.Key("a"), roachpb.Key("z")} get := &roachpb.GetRequest{ Span: roachpb.Span{Key: writes[0]}, } get.EndKey = writes[len(writes)-1] if _, err := client.SendWrapped(tds, nil, get); err == nil { t.Errorf("able to call Get with a key range: %v", get) } var delTS roachpb.Timestamp for i, k := range writes { put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k)) reply, err := client.SendWrapped(tds, nil, put) if err != nil { t.Fatal(err) } scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next(), 0).(*roachpb.ScanRequest) // The Put ts may have been pushed by tsCache, // so make sure we see their values in our Scan. delTS = reply.(*roachpb.PutResponse).Timestamp reply, err = client.SendWrappedWith(tds, nil, roachpb.Header{Timestamp: delTS}, scan) if err != nil { t.Fatal(err) } sr := reply.(*roachpb.ScanResponse) if sr.Txn != nil { // This was the other way around at some point in the past. // Same below for Delete, etc. t.Errorf("expected no transaction in response header") } if rows := sr.Rows; len(rows) != i+1 { t.Fatalf("expected %d rows, but got %d", i+1, len(rows)) } } del := &roachpb.DeleteRangeRequest{ Span: roachpb.Span{ Key: writes[0], EndKey: roachpb.Key(writes[len(writes)-1]).Next(), }, } reply, err := client.SendWrappedWith(tds, nil, roachpb.Header{Timestamp: delTS}, del) if err != nil { t.Fatal(err) } dr := reply.(*roachpb.DeleteRangeResponse) if dr.Txn != nil { t.Errorf("expected no transaction in response header") } if n := dr.NumDeleted; n != int64(len(writes)) { t.Errorf("expected %d keys to be deleted, but got %d instead", len(writes), n) } scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next(), 0).(*roachpb.ScanRequest) txn := &roachpb.Transaction{Name: "MyTxn"} reply, err = client.SendWrappedWith(tds, nil, roachpb.Header{Txn: txn}, scan) if err != nil { t.Fatal(err) } sr := reply.(*roachpb.ScanResponse) if txn := sr.Txn; txn == nil || txn.Name != "MyTxn" { t.Errorf("wanted Txn to persist, but it changed to %v", txn) } if rows := sr.Rows; len(rows) > 0 { t.Fatalf("scan after delete returned rows: %v", rows) } }
// TestMultiRangeScanReverseScanInconsistent verifies that a Scan/ReverseScan // across ranges that doesn't require read consistency will set a timestamp // using the clock local to the distributed sender. func TestMultiRangeScanReverseScanInconsistent(t *testing.T) { defer leaktest.AfterTest(t) s := server.StartTestServer(t) defer s.Stop() db := setupMultipleRanges(t, s, "b") // Write keys "a" and "b", the latter of which is the first key in the // second range. keys := [2]string{"a", "b"} ts := [2]time.Time{} for i, key := range keys { b := &client.Batch{} b.Put(key, "value") if pErr := db.Run(b); pErr != nil { t.Fatal(pErr) } ts[i] = b.Results[0].Rows[0].Timestamp() log.Infof("%d: %d.%d", i, ts[i].Unix(), ts[i].Nanosecond()) if i == 0 { util.SucceedsWithin(t, time.Second, func() error { // Enforce that when we write the second key, it's written // with a strictly higher timestamp. We're dropping logical // ticks and the clock may just have been pushed into the // future, so that's necessary. See #3122. if !s.Clock().PhysicalTime().After(ts[0]) { return errors.New("time stands still") } return nil }) } } // Do an inconsistent Scan/ReverseScan from a new DistSender and verify // it does the read at its local clock and doesn't receive an // OpRequiresTxnError. We set the local clock to the timestamp of // the first key to verify it's used to read only key "a". manual := hlc.NewManualClock(ts[1].UnixNano() - 1) clock := hlc.NewClock(manual.UnixNano) ds := kv.NewDistSender(&kv.DistSenderContext{Clock: clock, RPCContext: s.RPCContext()}, s.Gossip()) // Scan. sa := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("c"), 0).(*roachpb.ScanRequest) reply, pErr := client.SendWrappedWith(ds, nil, roachpb.Header{ ReadConsistency: roachpb.INCONSISTENT, }, sa) if pErr != nil { t.Fatal(pErr) } sr := reply.(*roachpb.ScanResponse) if l := len(sr.Rows); l != 1 { t.Fatalf("expected 1 row; got %d", l) } if key := string(sr.Rows[0].Key); keys[0] != key { t.Errorf("expected key %q; got %q", keys[0], key) } // ReverseScan. rsa := roachpb.NewReverseScan(roachpb.Key("a"), roachpb.Key("c"), 0).(*roachpb.ReverseScanRequest) reply, pErr = client.SendWrappedWith(ds, nil, roachpb.Header{ ReadConsistency: roachpb.INCONSISTENT, }, rsa) if pErr != nil { t.Fatal(pErr) } rsr := reply.(*roachpb.ReverseScanResponse) if l := len(rsr.Rows); l != 1 { t.Fatalf("expected 1 row; got %d", l) } if key := string(rsr.Rows[0].Key); keys[0] != key { t.Errorf("expected key %q; got %q", keys[0], key) } }
// TestMultiRangeScanReverseScanInconsistent verifies that a Scan/ReverseScan // across ranges that doesn't require read consistency will set a timestamp // using the clock local to the distributed sender. func TestMultiRangeScanReverseScanInconsistent(t *testing.T) { defer leaktest.AfterTest(t) s, db := setupMultipleRanges(t, "b") defer s.Stop() // Write keys "a" and "b", the latter of which is the first key in the // second range. keys := [2]string{"a", "b"} ts := [2]time.Time{} // This outer loop may seem awkward, but we've actually had issue #3122 // since the two timestamps ended up equal. This can happen (very rarely) // since both timestamps are HLC internally and then have their logical // component dropped. Lease changes can push the HLC ahead of the wall // time for short amounts of time, so that losing the logical tick matters. for !ts[1].After(ts[0]) { for i, key := range keys { b := &client.Batch{} b.Put(key, "value") if err := db.Run(b); err != nil { t.Fatal(err) } ts[i] = b.Results[0].Rows[0].Timestamp() log.Infof("%d: %s", i, b.Results[0].Rows[0].Timestamp()) } } // Do an inconsistent Scan/ReverseScan from a new DistSender and verify // it does the read at its local clock and doesn't receive an // OpRequiresTxnError. We set the local clock to the timestamp of // the first key to verify it's used to read only key "a". manual := hlc.NewManualClock(ts[1].UnixNano() - 1) clock := hlc.NewClock(manual.UnixNano) ds := kv.NewDistSender(&kv.DistSenderContext{Clock: clock}, s.Gossip()) // Scan. sa := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("c"), 0).(*roachpb.ScanRequest) reply, err := client.SendWrappedWith(ds, nil, roachpb.Header{ ReadConsistency: roachpb.INCONSISTENT, }, sa) if err != nil { t.Fatal(err) } sr := reply.(*roachpb.ScanResponse) if l := len(sr.Rows); l != 1 { t.Fatalf("expected 1 row; got %d", l) } if key := string(sr.Rows[0].Key); keys[0] != key { t.Errorf("expected key %q; got %q", keys[0], key) } // ReverseScan. rsa := roachpb.NewReverseScan(roachpb.Key("a"), roachpb.Key("c"), 0).(*roachpb.ReverseScanRequest) reply, err = client.SendWrappedWith(ds, nil, roachpb.Header{ ReadConsistency: roachpb.INCONSISTENT, }, rsa) if err != nil { t.Fatal(err) } rsr := reply.(*roachpb.ReverseScanResponse) if l := len(rsr.Rows); l != 1 { t.Fatalf("expected 1 row; got %d", l) } if key := string(rsr.Rows[0].Key); keys[0] != key { t.Errorf("expected key %q; got %q", keys[0], key) } }
// TestMultiRangeScanDeleteRange tests that commands which access multiple // ranges are carried out properly. func TestMultiRangeScanDeleteRange(t *testing.T) { defer leaktest.AfterTest(t)() s, _, _ := serverutils.StartServer(t, base.TestServerArgs{}) defer s.Stopper().Stop() ts := s.(*TestServer) retryOpts := base.DefaultRetryOptions() retryOpts.Closer = ts.stopper.ShouldQuiesce() ds := kv.NewDistSender(&kv.DistSenderConfig{ Clock: s.Clock(), RPCContext: s.RPCContext(), RPCRetryOptions: &retryOpts, }, ts.Gossip()) ctx := tracing.WithTracer(context.Background(), tracing.NewTracer()) tds := kv.NewTxnCoordSender(ctx, ds, s.Clock(), ts.Ctx.Linearizable, ts.stopper, kv.MakeTxnMetrics()) if err := ts.node.ctx.DB.AdminSplit("m"); err != nil { t.Fatal(err) } writes := []roachpb.Key{roachpb.Key("a"), roachpb.Key("z")} get := &roachpb.GetRequest{ Span: roachpb.Span{Key: writes[0]}, } get.EndKey = writes[len(writes)-1] if _, err := client.SendWrapped(tds, nil, get); err == nil { t.Errorf("able to call Get with a key range: %v", get) } var delTS hlc.Timestamp for i, k := range writes { put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k)) reply, err := client.SendWrapped(tds, nil, put) if err != nil { t.Fatal(err) } scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next()) reply, err = client.SendWrapped(tds, nil, scan) if err != nil { t.Fatal(err) } sr := reply.(*roachpb.ScanResponse) if sr.Txn != nil { // This was the other way around at some point in the past. // Same below for Delete, etc. t.Errorf("expected no transaction in response header") } if rows := sr.Rows; len(rows) != i+1 { t.Fatalf("expected %d rows, but got %d", i+1, len(rows)) } } del := &roachpb.DeleteRangeRequest{ Span: roachpb.Span{ Key: writes[0], EndKey: roachpb.Key(writes[len(writes)-1]).Next(), }, ReturnKeys: true, } reply, err := client.SendWrappedWith(tds, nil, roachpb.Header{Timestamp: delTS}, del) if err != nil { t.Fatal(err) } dr := reply.(*roachpb.DeleteRangeResponse) if dr.Txn != nil { t.Errorf("expected no transaction in response header") } if !reflect.DeepEqual(dr.Keys, writes) { t.Errorf("expected %d keys to be deleted, but got %d instead", writes, dr.Keys) } scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next()) txn := &roachpb.Transaction{Name: "MyTxn"} reply, err = client.SendWrappedWith(tds, nil, roachpb.Header{Txn: txn}, scan) if err != nil { t.Fatal(err) } sr := reply.(*roachpb.ScanResponse) if txn := sr.Txn; txn == nil || txn.Name != "MyTxn" { t.Errorf("wanted Txn to persist, but it changed to %v", txn) } if rows := sr.Rows; len(rows) > 0 { t.Fatalf("scan after delete returned rows: %v", rows) } }
// TestMultiRangeScanReverseScanInconsistent verifies that a Scan/ReverseScan // across ranges that doesn't require read consistency will set a timestamp // using the clock local to the distributed sender. func TestMultiRangeScanReverseScanInconsistent(t *testing.T) { defer leaktest.AfterTest(t)() s := server.StartTestServer(t) defer s.Stop() db := setupMultipleRanges(t, s, "b") // Write keys "a" and "b", the latter of which is the first key in the // second range. keys := [2]string{"a", "b"} ts := [2]roachpb.Timestamp{} for i, key := range keys { b := &client.Batch{} b.Put(key, "value") if err := db.Run(b); err != nil { t.Fatal(err) } ts[i] = s.Clock().Now() log.Infof("%d: %d", i, ts[i]) if i == 0 { util.SucceedsSoon(t, func() error { // Enforce that when we write the second key, it's written // with a strictly higher timestamp. We're dropping logical // ticks and the clock may just have been pushed into the // future, so that's necessary. See #3122. if !ts[0].Less(s.Clock().Now()) { return errors.New("time stands still") } return nil }) } } // Do an inconsistent Scan/ReverseScan from a new DistSender and verify // it does the read at its local clock and doesn't receive an // OpRequiresTxnError. We set the local clock to the timestamp of // just above the first key to verify it's used to read only key "a". for _, request := range []roachpb.Request{ roachpb.NewScan(roachpb.Key("a"), roachpb.Key("c"), 0), roachpb.NewReverseScan(roachpb.Key("a"), roachpb.Key("c"), 0), } { manual := hlc.NewManualClock(ts[0].WallTime + 1) clock := hlc.NewClock(manual.UnixNano) ds := kv.NewDistSender(&kv.DistSenderContext{Clock: clock, RPCContext: s.RPCContext()}, s.Gossip()) reply, err := client.SendWrappedWith(ds, nil, roachpb.Header{ ReadConsistency: roachpb.INCONSISTENT, }, request) if err != nil { t.Fatal(err) } var rows []roachpb.KeyValue switch r := reply.(type) { case *roachpb.ScanResponse: rows = r.Rows case *roachpb.ReverseScanResponse: rows = r.Rows default: t.Fatalf("unexpected response %T: %v", reply, reply) } if l := len(rows); l != 1 { t.Fatalf("expected 1 row; got %d", l) } if key := string(rows[0].Key); keys[0] != key { t.Errorf("expected key %q; got %q", keys[0], key) } } }
// TestTruncateWithLocalSpanAndDescriptor verifies that a batch request with local keys // is truncated with a range span and the range of a descriptor found in cache. func TestTruncateWithLocalSpanAndDescriptor(t *testing.T) { defer leaktest.AfterTest(t)() g, s := makeTestGossip(t) defer s() if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil { t.Fatal(err) } nd := &roachpb.NodeDescriptor{ NodeID: roachpb.NodeID(1), Address: util.MakeUnresolvedAddr(testAddress.Network(), testAddress.String()), } if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(1)), nd, time.Hour); err != nil { t.Fatal(err) } // Fill mockRangeDescriptorDB with two descriptors. var descriptor1 = roachpb.RangeDescriptor{ RangeID: 1, StartKey: roachpb.RKeyMin, EndKey: roachpb.RKey("b"), Replicas: []roachpb.ReplicaDescriptor{ { NodeID: 1, StoreID: 1, }, }, } var descriptor2 = roachpb.RangeDescriptor{ RangeID: 2, StartKey: roachpb.RKey("b"), EndKey: roachpb.RKey("c"), Replicas: []roachpb.ReplicaDescriptor{ { NodeID: 1, StoreID: 1, }, }, } var descriptor3 = roachpb.RangeDescriptor{ RangeID: 3, StartKey: roachpb.RKey("c"), EndKey: roachpb.RKeyMax, Replicas: []roachpb.ReplicaDescriptor{ { NodeID: 1, StoreID: 1, }, }, } descDB := mockRangeDescriptorDB(func(key roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) { switch { case !key.Less(roachpb.RKey("c")): return []roachpb.RangeDescriptor{descriptor3}, nil case !key.Less(roachpb.RKey("b")): return []roachpb.RangeDescriptor{descriptor2}, nil default: return []roachpb.RangeDescriptor{descriptor1}, nil } }) // Define our rpcSend stub which checks the span of the batch // requests. requests := 0 sendStub := func(_ SendOptions, _ ReplicaSlice, ba roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) { h := ba.Requests[0].GetInner().Header() switch requests { case 0: wantStart := keys.RangeDescriptorKey(roachpb.RKey("a")) wantEnd := keys.MakeRangeKeyPrefix(roachpb.RKey("b")) if !(h.Key.Equal(wantStart) && h.EndKey.Equal(wantEnd)) { t.Errorf("Unexpected span [%s,%s), want [%s,%s)", h.Key, h.EndKey, wantStart, wantEnd) } case 1: wantStart := keys.MakeRangeKeyPrefix(roachpb.RKey("b")) wantEnd := keys.MakeRangeKeyPrefix(roachpb.RKey("c")) if !(h.Key.Equal(wantStart) && h.EndKey.Equal(wantEnd)) { t.Errorf("Unexpected span [%s,%s), want [%s,%s)", h.Key, h.EndKey, wantStart, wantEnd) } case 2: wantStart := keys.MakeRangeKeyPrefix(roachpb.RKey("c")) wantEnd := keys.RangeDescriptorKey(roachpb.RKey("c")) if !(h.Key.Equal(wantStart) && h.EndKey.Equal(wantEnd)) { t.Errorf("Unexpected span [%s,%s), want [%s,%s)", h.Key, h.EndKey, wantStart, wantEnd) } } requests++ batchReply := &roachpb.BatchResponse{} reply := &roachpb.ScanResponse{} batchReply.Add(reply) return batchReply, nil } ctx := &DistSenderContext{ RPCSend: sendStub, RangeDescriptorDB: descDB, } ds := NewDistSender(ctx, g) // Send a batch request contains two scans. In the first // attempt, the range of the descriptor found in the cache is // ["", "b"). The request is truncated to contain only the scan // on local keys that address up to "b". // // In the second attempt, The range of the descriptor found in // the cache is ["b", "d"), The request is truncated to contain // only the scan on local keys that address from "b" to "d". ba := roachpb.BatchRequest{} ba.Txn = &roachpb.Transaction{Name: "test"} ba.Add(roachpb.NewScan(keys.RangeDescriptorKey(roachpb.RKey("a")), keys.RangeDescriptorKey(roachpb.RKey("c")), 0)) if _, pErr := ds.Send(context.Background(), ba); pErr != nil { t.Fatal(pErr) } if want := 3; requests != want { t.Errorf("expected request to be split into %d parts, found %d", want, requests) } }