// CopyInto copies all the cached results from this response cache // into the destRangeID response cache. Failures decoding individual // cache entries return an error. func (rc *ResponseCache) CopyInto(e engine.Engine, destRangeID roachpb.RangeID) error { start := engine.MVCCEncodeKey( keys.ResponseCacheKey(rc.rangeID, roachpb.KeyMin)) end := engine.MVCCEncodeKey( keys.ResponseCacheKey(rc.rangeID, roachpb.KeyMax)) return e.Iterate(start, end, func(kv engine.MVCCKeyValue) (bool, error) { // Decode the key into a cmd, skipping on error. Otherwise, // write it to the corresponding key in the new cache. family, err := rc.decodeResponseCacheKey(kv.Key) if err != nil { return false, util.Errorf("could not decode a response cache key %s: %s", roachpb.Key(kv.Key), err) } key := keys.ResponseCacheKey(destRangeID, family) encKey := engine.MVCCEncodeKey(key) // Decode the value, update the checksum and re-encode. meta := &engine.MVCCMetadata{} if err := proto.Unmarshal(kv.Value, meta); err != nil { return false, util.Errorf("could not decode response cache value %s [% x]: %s", roachpb.Key(kv.Key), kv.Value, err) } meta.Value.Checksum = nil meta.Value.InitChecksum(key) _, _, err = engine.PutProto(e, encKey, meta) return false, err }) }
// CopyFrom copies all the cached results from the originRangeID // response cache into this one. Note that the cache will not be // locked while copying is in progress. Failures decoding individual // cache entries return an error. The copy is done directly using the // engine instead of interpreting values through MVCC for efficiency. func (rc *ResponseCache) CopyFrom(e engine.Engine, originRangeID proto.RangeID) error { prefix := keys.ResponseCacheKey(originRangeID, nil) // response cache prefix start := engine.MVCCEncodeKey(prefix) end := engine.MVCCEncodeKey(prefix.PrefixEnd()) return e.Iterate(start, end, func(kv proto.RawKeyValue) (bool, error) { // Decode the key into a cmd, skipping on error. Otherwise, // write it to the corresponding key in the new cache. cmdID, err := rc.decodeResponseCacheKey(kv.Key) if err != nil { return false, util.Errorf("could not decode a response cache key %s: %s", proto.Key(kv.Key), err) } key := keys.ResponseCacheKey(rc.rangeID, &cmdID) encKey := engine.MVCCEncodeKey(key) // Decode the value, update the checksum and re-encode. meta := &engine.MVCCMetadata{} if err := gogoproto.Unmarshal(kv.Value, meta); err != nil { return false, util.Errorf("could not decode response cache value %s [% x]: %s", proto.Key(kv.Key), kv.Value, err) } meta.Value.Checksum = nil meta.Value.InitChecksum(key) _, _, err = engine.PutProto(e, encKey, meta) return false, err }) }
// TestRocksDBCompaction verifies that a garbage collector can be // installed on a RocksDB engine and will properly compact response // cache and transaction entries. func TestRocksDBCompaction(t *testing.T) { defer leaktest.AfterTest(t) gob.Register(proto.Timestamp{}) rocksdb := newMemRocksDB(proto.Attributes{Attrs: []string{"ssd"}}, testCacheSize) err := rocksdb.Open() if err != nil { t.Fatalf("could not create new in-memory rocksdb db instance: %v", err) } rocksdb.SetGCTimeouts(1, 2) defer rocksdb.Close() cmdID := &proto.ClientCmdID{WallTime: 1, Random: 1} // Write two transaction values and two response cache values such // that exactly one of each should be GC'd based on our GC timeouts. kvs := []proto.KeyValue{ { Key: keys.ResponseCacheKey(1, cmdID), Value: proto.Value{Bytes: encodePutResponse(makeTS(2, 0), t)}, }, { Key: keys.ResponseCacheKey(2, cmdID), Value: proto.Value{Bytes: encodePutResponse(makeTS(3, 0), t)}, }, { Key: keys.TransactionKey(proto.Key("a"), proto.Key(uuid.NewUUID4())), Value: proto.Value{Bytes: encodeTransaction(makeTS(1, 0), t)}, }, { Key: keys.TransactionKey(proto.Key("b"), proto.Key(uuid.NewUUID4())), Value: proto.Value{Bytes: encodeTransaction(makeTS(2, 0), t)}, }, } for _, kv := range kvs { if err := MVCCPut(rocksdb, nil, kv.Key, proto.ZeroTimestamp, kv.Value, nil); err != nil { t.Fatal(err) } } // Compact range and scan remaining values to compare. rocksdb.CompactRange(nil, nil) actualKVs, _, err := MVCCScan(rocksdb, proto.KeyMin, proto.KeyMax, 0, proto.ZeroTimestamp, true, nil) if err != nil { t.Fatalf("could not run scan: %v", err) } var keys []proto.Key for _, kv := range actualKVs { keys = append(keys, kv.Key) } expKeys := []proto.Key{ kvs[1].Key, kvs[3].Key, } if !reflect.DeepEqual(expKeys, keys) { t.Errorf("expected keys %+v, got keys %+v", expKeys, keys) } }
// createRangeData creates sample range data in all possible areas of // the key space. Returns a slice of the encoded keys of all created // data. func createRangeData(r *Replica, t *testing.T) []roachpb.EncodedKey { ts0 := roachpb.ZeroTimestamp ts := roachpb.Timestamp{WallTime: 1} keyTSs := []struct { key roachpb.Key ts roachpb.Timestamp }{ {keys.ResponseCacheKey(r.Desc().RangeID, &roachpb.ClientCmdID{WallTime: 1, Random: 1}), ts0}, {keys.ResponseCacheKey(r.Desc().RangeID, &roachpb.ClientCmdID{WallTime: 2, Random: 2}), ts0}, {keys.RaftHardStateKey(r.Desc().RangeID), ts0}, {keys.RaftLogKey(r.Desc().RangeID, 1), ts0}, {keys.RaftLogKey(r.Desc().RangeID, 2), ts0}, {keys.RangeGCMetadataKey(r.Desc().RangeID), ts0}, {keys.RangeLastVerificationTimestampKey(r.Desc().RangeID), ts0}, {keys.RangeStatsKey(r.Desc().RangeID), ts0}, {keys.RangeDescriptorKey(r.Desc().StartKey), ts}, {keys.TransactionKey(roachpb.Key(r.Desc().StartKey), []byte("1234")), ts0}, {keys.TransactionKey(roachpb.Key(r.Desc().StartKey.Next()), []byte("5678")), ts0}, {keys.TransactionKey(fakePrevKey(r.Desc().EndKey), []byte("2468")), ts0}, // TODO(bdarnell): KeyMin.Next() results in a key in the reserved system-local space. // Once we have resolved https://github.com/cockroachdb/cockroach/issues/437, // replace this with something that reliably generates the first valid key in the range. //{r.Desc().StartKey.Next(), ts}, // The following line is similar to StartKey.Next() but adds more to the key to // avoid falling into the system-local space. {append(append([]byte{}, r.Desc().StartKey...), '\x01'), ts}, {fakePrevKey(r.Desc().EndKey), ts}, } keys := []roachpb.EncodedKey{} for _, keyTS := range keyTSs { if err := engine.MVCCPut(r.store.Engine(), nil, keyTS.key, keyTS.ts, roachpb.MakeValueFromString("value"), nil); err != nil { t.Fatal(err) } keys = append(keys, engine.MVCCEncodeKey(keyTS.key)) if !keyTS.ts.Equal(ts0) { keys = append(keys, engine.MVCCEncodeVersionKey(keyTS.key, keyTS.ts)) } } return keys }
// PutSequence writes a sequence number for the specified family. func (rc *ResponseCache) PutSequence(e engine.Engine, family []byte, sequence int64, err error) error { if sequence <= 0 || len(family) == 0 { return errEmptyID } if !rc.shouldCacheError(err) { return nil } // Write the response value to the engine. key := keys.ResponseCacheKey(rc.rangeID, family) var v roachpb.Value v.SetInt(sequence) return engine.MVCCPut(e, nil /* ms */, key, roachpb.ZeroTimestamp, v, nil /* txn */) }
// GetResponse looks up a response matching the specified cmdID and // returns true if found. The response is deserialized into the // supplied reply parameter. If no response is found, returns // false. If a command is pending already for the cmdID, then this // method will block until the the command is completed or the // response cache is cleared. func (rc *ResponseCache) GetResponse(e engine.Engine, cmdID proto.ClientCmdID, reply proto.Response) (bool, error) { // Do nothing if command ID is empty. if cmdID.IsEmpty() { return false, nil } // Pull response from the cache and read into reply if available. rwResp := proto.ReadWriteCmdResponse{} key := keys.ResponseCacheKey(rc.raftID, &cmdID) ok, err := engine.MVCCGetProto(e, key, proto.ZeroTimestamp, true, nil, &rwResp) if ok && err == nil { gogoproto.Merge(reply, rwResp.GetValue().(gogoproto.Message)) } return ok, err }
// GetResponse looks up a response matching the specified cmdID and // returns true if found. The response is deserialized into the // supplied reply parameter. If no response is found, returns // false. If a command is pending already for the cmdID, then this // method will block until the the command is completed or the // response cache is cleared. func (rc *ResponseCache) GetResponse(cmdID proto.ClientCmdID, reply proto.Response) (bool, error) { // Do nothing if command ID is empty. if cmdID.IsEmpty() { return false, nil } // If the response is in the cache or we experienced an error, return. rwResp := proto.ReadWriteCmdResponse{} key := keys.ResponseCacheKey(rc.raftID, &cmdID) ok, err := engine.MVCCGetProto(rc.engine, key, proto.ZeroTimestamp, true, nil, &rwResp) if ok && err == nil { gogoproto.Merge(reply, rwResp.GetValue().(gogoproto.Message)) } return ok, err }
// GetSequence looks up the latest sequence number recorded for this family. On a // cache miss, zero is returned. func (rc *ResponseCache) GetSequence(e engine.Engine, family []byte) (int64, error) { if len(family) == 0 { return 0, errEmptyID } // Pull response from the cache and read into reply if available. key := keys.ResponseCacheKey(rc.rangeID, family) v, _, err := engine.MVCCGet(e, key, roachpb.ZeroTimestamp, true, nil) if err != nil { return 0, err } if v == nil { return 0, nil } return v.GetInt() }
// PutResponse writes a response to the cache for the specified cmdID. func (rc *ResponseCache) PutResponse(e engine.Engine, cmdID proto.ClientCmdID, reply proto.Response) error { // Do nothing if command ID is empty. if cmdID.IsEmpty() { return nil } // Write the response value to the engine. var err error if rc.shouldCacheResponse(reply) { key := keys.ResponseCacheKey(rc.raftID, &cmdID) rwResp := &proto.ReadWriteCmdResponse{} if !rwResp.SetValue(reply) { log.Fatalf("response %T not supported by response cache", reply) } err = engine.MVCCPutProto(e, nil, key, proto.ZeroTimestamp, nil, rwResp) } return err }
// PutResponse writes a response and an error associated with it to the // cache for the specified cmdID. func (rc *ResponseCache) PutResponse(e engine.Engine, cmdID roachpb.ClientCmdID, replyWithErr roachpb.ResponseWithError) error { // Do nothing if command ID is empty. if cmdID.IsEmpty() { return nil } // Write the response value to the engine. if rc.shouldCacheResponse(replyWithErr) { // Write the error into the reply before caching. replyWithErr.Reply.SetGoError(replyWithErr.Err) // Be sure to clear it when you're done! defer func() { replyWithErr.Reply.BatchResponse_Header.Error = nil }() key := keys.ResponseCacheKey(rc.rangeID, &cmdID) return engine.MVCCPutProto(e, nil, key, roachpb.ZeroTimestamp, nil, replyWithErr.Reply) } return nil }
// GetResponse looks up a response matching the specified cmdID. If the // response is found, it is returned along with its associated error. // If the response is not found, nil is returned for both the response // and its error. In all cases, the third return value is the error // returned from the engine when reading the on-disk cache. func (rc *ResponseCache) GetResponse(e engine.Engine, cmdID proto.ClientCmdID) (proto.ResponseWithError, error) { // Do nothing if command ID is empty. if cmdID.IsEmpty() { return proto.ResponseWithError{}, nil } // Pull response from the cache and read into reply if available. br := &proto.BatchResponse{} key := keys.ResponseCacheKey(rc.rangeID, &cmdID) ok, err := engine.MVCCGetProto(e, key, proto.ZeroTimestamp, true, nil, br) if err != nil { return proto.ResponseWithError{}, err } if ok { header := br.Header() defer func() { header.Error = nil }() return proto.ResponseWithError{Reply: br, Err: header.GoError()}, nil } return proto.ResponseWithError{}, nil }
// GetResponse looks up a response matching the specified cmdID. If the // response is found, it is returned along with its associated error. // If the response is not found, nil is returned for both the response // and its error. In all cases, the third return value is the error // returned from the engine when reading the on-disk cache. func (rc *ResponseCache) GetResponse(e engine.Engine, cmdID proto.ClientCmdID) (proto.ResponseWithError, error) { // Do nothing if command ID is empty. if cmdID.IsEmpty() { return proto.ResponseWithError{}, nil } // Pull response from the cache and read into reply if available. var rwResp proto.ReadWriteCmdResponse key := keys.ResponseCacheKey(rc.raftID, &cmdID) ok, err := engine.MVCCGetProto(e, key, proto.ZeroTimestamp, true, nil, &rwResp) if err != nil { return proto.ResponseWithError{}, err } if ok { resp := rwResp.GetValue().(proto.Response) header := resp.Header() defer func() { header.Error = nil }() return proto.ResponseWithError{resp, header.GoError()}, nil } return proto.ResponseWithError{}, nil }
// PutResponse writes a response and an error associated with it to the // cache for the specified cmdID. func (rc *ResponseCache) PutResponse(e engine.Engine, cmdID proto.ClientCmdID, replyWithErr proto.ResponseWithError) error { // Do nothing if command ID is empty. if cmdID.IsEmpty() { return nil } // Write the response value to the engine. if rc.shouldCacheResponse(replyWithErr) { // Write the error into the reply before caching. header := replyWithErr.Reply.Header() header.SetGoError(replyWithErr.Err) // Be sure to clear it when you're done! defer func() { header.Error = nil }() key := keys.ResponseCacheKey(rc.raftID, &cmdID) var rwResp proto.ReadWriteCmdResponse if !rwResp.SetValue(replyWithErr.Reply) { panic(fmt.Sprintf("response %T not supported by response cache", replyWithErr.Reply)) } return engine.MVCCPutProto(e, nil, key, proto.ZeroTimestamp, nil, &rwResp) } return nil }
// ClearData removes all items stored in the persistent cache. It does not alter // the inflight map. func (rc *ResponseCache) ClearData(e engine.Engine) error { p := keys.ResponseCacheKey(rc.rangeID, nil) // prefix for all response cache entries with this range ID end := p.PrefixEnd() _, err := engine.ClearRange(e, engine.MVCCEncodeKey(p), engine.MVCCEncodeKey(end)) return err }
// ClearData removes all items stored in the persistent cache. func (rc *ResponseCache) ClearData(e engine.Engine) error { from := keys.ResponseCacheKey(rc.rangeID, roachpb.KeyMin) to := keys.ResponseCacheKey(rc.rangeID, roachpb.KeyMax) _, err := engine.ClearRange(e, engine.MVCCEncodeKey(from), engine.MVCCEncodeKey(to)) return err }