// CopyInto copies all the cached results from this response cache
// into the destRangeID response cache. Failures decoding individual
// cache entries return an error.
func (rc *ResponseCache) CopyInto(e engine.Engine, destRangeID roachpb.RangeID) error {
start := engine.MVCCEncodeKey(
keys.ResponseCacheKey(rc.rangeID, roachpb.KeyMin))
end := engine.MVCCEncodeKey(
keys.ResponseCacheKey(rc.rangeID, roachpb.KeyMax))
return e.Iterate(start, end, func(kv engine.MVCCKeyValue) (bool, error) {
// Decode the key into a cmd, skipping on error. Otherwise,
// write it to the corresponding key in the new cache.
family, err := rc.decodeResponseCacheKey(kv.Key)
if err != nil {
return false, util.Errorf("could not decode a response cache key %s: %s",
roachpb.Key(kv.Key), err)
}
key := keys.ResponseCacheKey(destRangeID, family)
encKey := engine.MVCCEncodeKey(key)
// Decode the value, update the checksum and re-encode.
meta := &engine.MVCCMetadata{}
if err := proto.Unmarshal(kv.Value, meta); err != nil {
return false, util.Errorf("could not decode response cache value %s [% x]: %s",
roachpb.Key(kv.Key), kv.Value, err)
}
meta.Value.Checksum = nil
meta.Value.InitChecksum(key)
_, _, err = engine.PutProto(e, encKey, meta)
return false, err
})
}
// CopyFrom copies all the cached results from the originRangeID
// response cache into this one. Note that the cache will not be
// locked while copying is in progress. Failures decoding individual
// cache entries return an error. The copy is done directly using the
// engine instead of interpreting values through MVCC for efficiency.
func (rc *ResponseCache) CopyFrom(e engine.Engine, originRangeID proto.RangeID) error {
prefix := keys.ResponseCacheKey(originRangeID, nil) // response cache prefix
start := engine.MVCCEncodeKey(prefix)
end := engine.MVCCEncodeKey(prefix.PrefixEnd())
return e.Iterate(start, end, func(kv proto.RawKeyValue) (bool, error) {
// Decode the key into a cmd, skipping on error. Otherwise,
// write it to the corresponding key in the new cache.
cmdID, err := rc.decodeResponseCacheKey(kv.Key)
if err != nil {
return false, util.Errorf("could not decode a response cache key %s: %s",
proto.Key(kv.Key), err)
}
key := keys.ResponseCacheKey(rc.rangeID, &cmdID)
encKey := engine.MVCCEncodeKey(key)
// Decode the value, update the checksum and re-encode.
meta := &engine.MVCCMetadata{}
if err := gogoproto.Unmarshal(kv.Value, meta); err != nil {
return false, util.Errorf("could not decode response cache value %s [% x]: %s",
proto.Key(kv.Key), kv.Value, err)
}
meta.Value.Checksum = nil
meta.Value.InitChecksum(key)
_, _, err = engine.PutProto(e, encKey, meta)
return false, err
})
}
// TestRocksDBCompaction verifies that a garbage collector can be
// installed on a RocksDB engine and will properly compact response
// cache and transaction entries.
func TestRocksDBCompaction(t *testing.T) {
defer leaktest.AfterTest(t)
gob.Register(proto.Timestamp{})
rocksdb := newMemRocksDB(proto.Attributes{Attrs: []string{"ssd"}}, testCacheSize)
err := rocksdb.Open()
if err != nil {
t.Fatalf("could not create new in-memory rocksdb db instance: %v", err)
}
rocksdb.SetGCTimeouts(1, 2)
defer rocksdb.Close()
cmdID := &proto.ClientCmdID{WallTime: 1, Random: 1}
// Write two transaction values and two response cache values such
// that exactly one of each should be GC'd based on our GC timeouts.
kvs := []proto.KeyValue{
{
Key: keys.ResponseCacheKey(1, cmdID),
Value: proto.Value{Bytes: encodePutResponse(makeTS(2, 0), t)},
},
{
Key: keys.ResponseCacheKey(2, cmdID),
Value: proto.Value{Bytes: encodePutResponse(makeTS(3, 0), t)},
},
{
Key: keys.TransactionKey(proto.Key("a"), proto.Key(uuid.NewUUID4())),
Value: proto.Value{Bytes: encodeTransaction(makeTS(1, 0), t)},
},
{
Key: keys.TransactionKey(proto.Key("b"), proto.Key(uuid.NewUUID4())),
Value: proto.Value{Bytes: encodeTransaction(makeTS(2, 0), t)},
},
}
for _, kv := range kvs {
if err := MVCCPut(rocksdb, nil, kv.Key, proto.ZeroTimestamp, kv.Value, nil); err != nil {
t.Fatal(err)
}
}
// Compact range and scan remaining values to compare.
rocksdb.CompactRange(nil, nil)
actualKVs, _, err := MVCCScan(rocksdb, proto.KeyMin, proto.KeyMax,
0, proto.ZeroTimestamp, true, nil)
if err != nil {
t.Fatalf("could not run scan: %v", err)
}
var keys []proto.Key
for _, kv := range actualKVs {
keys = append(keys, kv.Key)
}
expKeys := []proto.Key{
kvs[1].Key,
kvs[3].Key,
}
if !reflect.DeepEqual(expKeys, keys) {
t.Errorf("expected keys %+v, got keys %+v", expKeys, keys)
}
}
// createRangeData creates sample range data in all possible areas of
// the key space. Returns a slice of the encoded keys of all created
// data.
func createRangeData(r *Replica, t *testing.T) []roachpb.EncodedKey {
ts0 := roachpb.ZeroTimestamp
ts := roachpb.Timestamp{WallTime: 1}
keyTSs := []struct {
key roachpb.Key
ts roachpb.Timestamp
}{
{keys.ResponseCacheKey(r.Desc().RangeID, &roachpb.ClientCmdID{WallTime: 1, Random: 1}), ts0},
{keys.ResponseCacheKey(r.Desc().RangeID, &roachpb.ClientCmdID{WallTime: 2, Random: 2}), ts0},
{keys.RaftHardStateKey(r.Desc().RangeID), ts0},
{keys.RaftLogKey(r.Desc().RangeID, 1), ts0},
{keys.RaftLogKey(r.Desc().RangeID, 2), ts0},
{keys.RangeGCMetadataKey(r.Desc().RangeID), ts0},
{keys.RangeLastVerificationTimestampKey(r.Desc().RangeID), ts0},
{keys.RangeStatsKey(r.Desc().RangeID), ts0},
{keys.RangeDescriptorKey(r.Desc().StartKey), ts},
{keys.TransactionKey(roachpb.Key(r.Desc().StartKey), []byte("1234")), ts0},
{keys.TransactionKey(roachpb.Key(r.Desc().StartKey.Next()), []byte("5678")), ts0},
{keys.TransactionKey(fakePrevKey(r.Desc().EndKey), []byte("2468")), ts0},
// TODO(bdarnell): KeyMin.Next() results in a key in the reserved system-local space.
// Once we have resolved https://github.com/cockroachdb/cockroach/issues/437,
// replace this with something that reliably generates the first valid key in the range.
//{r.Desc().StartKey.Next(), ts},
// The following line is similar to StartKey.Next() but adds more to the key to
// avoid falling into the system-local space.
{append(append([]byte{}, r.Desc().StartKey...), '\x01'), ts},
{fakePrevKey(r.Desc().EndKey), ts},
}
keys := []roachpb.EncodedKey{}
for _, keyTS := range keyTSs {
if err := engine.MVCCPut(r.store.Engine(), nil, keyTS.key, keyTS.ts, roachpb.MakeValueFromString("value"), nil); err != nil {
t.Fatal(err)
}
keys = append(keys, engine.MVCCEncodeKey(keyTS.key))
if !keyTS.ts.Equal(ts0) {
keys = append(keys, engine.MVCCEncodeVersionKey(keyTS.key, keyTS.ts))
}
}
return keys
}
// PutSequence writes a sequence number for the specified family.
func (rc *ResponseCache) PutSequence(e engine.Engine, family []byte, sequence int64, err error) error {
if sequence <= 0 || len(family) == 0 {
return errEmptyID
}
if !rc.shouldCacheError(err) {
return nil
}
// Write the response value to the engine.
key := keys.ResponseCacheKey(rc.rangeID, family)
var v roachpb.Value
v.SetInt(sequence)
return engine.MVCCPut(e, nil /* ms */, key, roachpb.ZeroTimestamp, v, nil /* txn */)
}
// GetResponse looks up a response matching the specified cmdID and
// returns true if found. The response is deserialized into the
// supplied reply parameter. If no response is found, returns
// false. If a command is pending already for the cmdID, then this
// method will block until the the command is completed or the
// response cache is cleared.
func (rc *ResponseCache) GetResponse(e engine.Engine, cmdID proto.ClientCmdID, reply proto.Response) (bool, error) {
// Do nothing if command ID is empty.
if cmdID.IsEmpty() {
return false, nil
}
// Pull response from the cache and read into reply if available.
rwResp := proto.ReadWriteCmdResponse{}
key := keys.ResponseCacheKey(rc.raftID, &cmdID)
ok, err := engine.MVCCGetProto(e, key, proto.ZeroTimestamp, true, nil, &rwResp)
if ok && err == nil {
gogoproto.Merge(reply, rwResp.GetValue().(gogoproto.Message))
}
return ok, err
}
// GetResponse looks up a response matching the specified cmdID and
// returns true if found. The response is deserialized into the
// supplied reply parameter. If no response is found, returns
// false. If a command is pending already for the cmdID, then this
// method will block until the the command is completed or the
// response cache is cleared.
func (rc *ResponseCache) GetResponse(cmdID proto.ClientCmdID, reply proto.Response) (bool, error) {
// Do nothing if command ID is empty.
if cmdID.IsEmpty() {
return false, nil
}
// If the response is in the cache or we experienced an error, return.
rwResp := proto.ReadWriteCmdResponse{}
key := keys.ResponseCacheKey(rc.raftID, &cmdID)
ok, err := engine.MVCCGetProto(rc.engine, key, proto.ZeroTimestamp, true, nil, &rwResp)
if ok && err == nil {
gogoproto.Merge(reply, rwResp.GetValue().(gogoproto.Message))
}
return ok, err
}
// GetSequence looks up the latest sequence number recorded for this family. On a
// cache miss, zero is returned.
func (rc *ResponseCache) GetSequence(e engine.Engine, family []byte) (int64, error) {
if len(family) == 0 {
return 0, errEmptyID
}
// Pull response from the cache and read into reply if available.
key := keys.ResponseCacheKey(rc.rangeID, family)
v, _, err := engine.MVCCGet(e, key, roachpb.ZeroTimestamp, true, nil)
if err != nil {
return 0, err
}
if v == nil {
return 0, nil
}
return v.GetInt()
}
// PutResponse writes a response to the cache for the specified cmdID.
func (rc *ResponseCache) PutResponse(e engine.Engine, cmdID proto.ClientCmdID, reply proto.Response) error {
// Do nothing if command ID is empty.
if cmdID.IsEmpty() {
return nil
}
// Write the response value to the engine.
var err error
if rc.shouldCacheResponse(reply) {
key := keys.ResponseCacheKey(rc.raftID, &cmdID)
rwResp := &proto.ReadWriteCmdResponse{}
if !rwResp.SetValue(reply) {
log.Fatalf("response %T not supported by response cache", reply)
}
err = engine.MVCCPutProto(e, nil, key, proto.ZeroTimestamp, nil, rwResp)
}
return err
}
// PutResponse writes a response and an error associated with it to the
// cache for the specified cmdID.
func (rc *ResponseCache) PutResponse(e engine.Engine, cmdID roachpb.ClientCmdID, replyWithErr roachpb.ResponseWithError) error {
// Do nothing if command ID is empty.
if cmdID.IsEmpty() {
return nil
}
// Write the response value to the engine.
if rc.shouldCacheResponse(replyWithErr) {
// Write the error into the reply before caching.
replyWithErr.Reply.SetGoError(replyWithErr.Err)
// Be sure to clear it when you're done!
defer func() { replyWithErr.Reply.BatchResponse_Header.Error = nil }()
key := keys.ResponseCacheKey(rc.rangeID, &cmdID)
return engine.MVCCPutProto(e, nil, key, roachpb.ZeroTimestamp, nil, replyWithErr.Reply)
}
return nil
}
// GetResponse looks up a response matching the specified cmdID. If the
// response is found, it is returned along with its associated error.
// If the response is not found, nil is returned for both the response
// and its error. In all cases, the third return value is the error
// returned from the engine when reading the on-disk cache.
func (rc *ResponseCache) GetResponse(e engine.Engine, cmdID proto.ClientCmdID) (proto.ResponseWithError, error) {
// Do nothing if command ID is empty.
if cmdID.IsEmpty() {
return proto.ResponseWithError{}, nil
}
// Pull response from the cache and read into reply if available.
br := &proto.BatchResponse{}
key := keys.ResponseCacheKey(rc.rangeID, &cmdID)
ok, err := engine.MVCCGetProto(e, key, proto.ZeroTimestamp, true, nil, br)
if err != nil {
return proto.ResponseWithError{}, err
}
if ok {
header := br.Header()
defer func() { header.Error = nil }()
return proto.ResponseWithError{Reply: br, Err: header.GoError()}, nil
}
return proto.ResponseWithError{}, nil
}
// GetResponse looks up a response matching the specified cmdID. If the
// response is found, it is returned along with its associated error.
// If the response is not found, nil is returned for both the response
// and its error. In all cases, the third return value is the error
// returned from the engine when reading the on-disk cache.
func (rc *ResponseCache) GetResponse(e engine.Engine, cmdID proto.ClientCmdID) (proto.ResponseWithError, error) {
// Do nothing if command ID is empty.
if cmdID.IsEmpty() {
return proto.ResponseWithError{}, nil
}
// Pull response from the cache and read into reply if available.
var rwResp proto.ReadWriteCmdResponse
key := keys.ResponseCacheKey(rc.raftID, &cmdID)
ok, err := engine.MVCCGetProto(e, key, proto.ZeroTimestamp, true, nil, &rwResp)
if err != nil {
return proto.ResponseWithError{}, err
}
if ok {
resp := rwResp.GetValue().(proto.Response)
header := resp.Header()
defer func() { header.Error = nil }()
return proto.ResponseWithError{resp, header.GoError()}, nil
}
return proto.ResponseWithError{}, nil
}
// PutResponse writes a response and an error associated with it to the
// cache for the specified cmdID.
func (rc *ResponseCache) PutResponse(e engine.Engine, cmdID proto.ClientCmdID, replyWithErr proto.ResponseWithError) error {
// Do nothing if command ID is empty.
if cmdID.IsEmpty() {
return nil
}
// Write the response value to the engine.
if rc.shouldCacheResponse(replyWithErr) {
// Write the error into the reply before caching.
header := replyWithErr.Reply.Header()
header.SetGoError(replyWithErr.Err)
// Be sure to clear it when you're done!
defer func() { header.Error = nil }()
key := keys.ResponseCacheKey(rc.raftID, &cmdID)
var rwResp proto.ReadWriteCmdResponse
if !rwResp.SetValue(replyWithErr.Reply) {
panic(fmt.Sprintf("response %T not supported by response cache", replyWithErr.Reply))
}
return engine.MVCCPutProto(e, nil, key, proto.ZeroTimestamp, nil, &rwResp)
}
return nil
}
// ClearData removes all items stored in the persistent cache. It does not alter
// the inflight map.
func (rc *ResponseCache) ClearData(e engine.Engine) error {
p := keys.ResponseCacheKey(rc.rangeID, nil) // prefix for all response cache entries with this range ID
end := p.PrefixEnd()
_, err := engine.ClearRange(e, engine.MVCCEncodeKey(p), engine.MVCCEncodeKey(end))
return err
}
// ClearData removes all items stored in the persistent cache.
func (rc *ResponseCache) ClearData(e engine.Engine) error {
from := keys.ResponseCacheKey(rc.rangeID, roachpb.KeyMin)
to := keys.ResponseCacheKey(rc.rangeID, roachpb.KeyMax)
_, err := engine.ClearRange(e, engine.MVCCEncodeKey(from), engine.MVCCEncodeKey(to))
return err
}