// executeCmd runs Store.ExecuteCmd in a goroutine. A channel with
// element type equal to the reply type is created and returned
// immediately. The reply is sent to the channel once the cmd has been
// executed by the store. The store is looked up from the store map
// if specified by header.Replica; otherwise, the command is being
// executed locally, and the replica is determined via lookup of
// header.Key in the ranges slice.
func (db *LocalDB) executeCmd(method string, header *storage.RequestHeader, args, reply interface{}) interface{} {
chanVal := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, reflect.TypeOf(reply)), 1)
replyVal := reflect.ValueOf(reply)
go func() {
// If the replica isn't specified in the header, look it up.
var err error
var store *storage.Store
// If we aren't given a Replica, then a little bending over backwards here. We need to find the Store, but all
// we have is the Key. So find its Range locally, and pull out its Replica which we use to find the Store.
// This lets us use the same codepath below (store.ExecuteCmd) for both locally and remotely originated
// commands.
if header.Replica.NodeID == 0 {
if repl := db.lookupReplica(header.Key); repl != nil {
header.Replica = *repl
} else {
err = util.Errorf("unable to lookup range replica for key %q", string(header.Key))
}
}
if err == nil {
store, err = db.GetStore(&header.Replica)
}
if err != nil {
reflect.Indirect(replyVal).FieldByName("Error").Set(reflect.ValueOf(err))
} else {
store.ExecuteCmd(method, header, args, reply)
}
chanVal.Send(replyVal)
}()
return chanVal.Interface()
}
// executeCmd synchronously runs Store.ExecuteCmd. The store is looked
// up from the store map if specified by header.Replica; otherwise,
// the command is being executed locally, and the replica is
// determined via lookup of header.Key in the ranges slice.
func (db *LocalDB) executeCmd(method string, args storage.Request, reply storage.Response) {
// If the replica isn't specified in the header, look it up.
var err error
var store *storage.Store
// If we aren't given a Replica, then a little bending over
// backwards here. We need to find the Store, but all we have is the
// Key. So find its Range locally, and pull out its Replica which we
// use to find the Store. This lets us use the same codepath below
// (store.ExecuteCmd) for both locally and remotely originated
// commands.
header := args.Header()
if header.Replica.NodeID == 0 {
if repl := db.lookupReplica(header.Key); repl != nil {
header.Replica = *repl
} else {
err = util.Errorf("unable to lookup range replica for key %q", string(header.Key))
}
}
if err == nil {
store, err = db.GetStore(&header.Replica)
}
if err != nil {
reply.Header().Error = err
} else {
store.ExecuteCmd(method, args, reply)
}
}
// ExecuteCmd synchronously runs Store.ExecuteCmd. The store is looked
// up from the store map if specified by header.Replica; otherwise,
// the command is being executed locally, and the replica is
// determined via lookup of header.Key in the ranges slice.
func (kv *LocalKV) ExecuteCmd(method string, args proto.Request, replyChan interface{}) {
// If the replica isn't specified in the header, look it up.
var err error
var store *storage.Store
// If we aren't given a Replica, then a little bending over
// backwards here. We need to find the Store, but all we have is the
// Key. So find its Range locally, and pull out its Replica which we
// use to find the Store. This lets us use the same codepath below
// (store.ExecuteCmd) for both locally and remotely originated
// commands.
header := args.Header()
if header.Replica.NodeID == 0 {
if repl := kv.lookupReplica(header.Key); repl != nil {
header.Replica = *repl
} else {
err = util.Errorf("unable to lookup range replica for key %q", string(header.Key))
}
}
if err == nil {
store, err = kv.GetStore(&header.Replica)
}
reply := reflect.New(reflect.TypeOf(replyChan).Elem().Elem()).Interface().(proto.Response)
if err != nil {
reply.Header().SetGoError(err)
} else {
store.ExecuteCmd(method, args, reply)
}
reflect.ValueOf(replyChan).Send(reflect.ValueOf(reply))
}
// executeCmd runs Store.ExecuteCmd in a goroutine. A channel with
// element type equal to the reply type is created and returned
// immediately. The reply is sent to the channel once the cmd has been
// executed by the store. The store is looked up from the store map
// if specified by header.Replica; otherwise, the command is being
// executed locally, and the replica is determined via lookup of
// header.Key in the ranges slice.
func (db *LocalDB) executeCmd(method string, header *storage.RequestHeader, args, reply interface{}) interface{} {
chanVal := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, reflect.TypeOf(reply)), 1)
replyVal := reflect.ValueOf(reply)
go func() {
// If the replica isn't specified in the header, look it up.
var err error
var store *storage.Store
if header.Replica.NodeID == 0 {
if repl := db.lookupReplica(header.Key); repl != nil {
header.Replica = *repl
} else {
err = util.Errorf("unable to lookup range replica for key %q", string(header.Key))
}
}
if err == nil {
store, err = db.GetStore(&header.Replica)
}
if err != nil {
reflect.Indirect(replyVal).FieldByName("Error").Set(reflect.ValueOf(err))
} else {
store.ExecuteCmd(method, header, args, reply)
}
chanVal.Send(replyVal)
}()
return chanVal.Interface()
}
// ExecuteCmd synchronously runs Store.ExecuteCmd. The store is looked
// up from the store map if specified by header.Replica; otherwise,
// the command is being executed locally, and the replica is
// determined via lookup through each of the stores.
func (kv *LocalKV) ExecuteCmd(method string, args proto.Request, replyChan interface{}) {
// If the replica isn't specified in the header, look it up.
var err error
var store *storage.Store
// If we aren't given a Replica, then a little bending over
// backwards here. We need to find the Store, but all we have is the
// Key. So find its Range locally. This lets us use the same
// codepath below (store.ExecuteCmd) for both locally and remotely
// originated commands.
header := args.Header()
if header.Replica.StoreID == 0 {
var repl *proto.Replica
repl, err = kv.lookupReplica(header.Key, header.EndKey)
if err == nil {
header.Replica = *repl
}
}
if err == nil {
store, err = kv.GetStore(header.Replica.StoreID)
}
reply := reflect.New(reflect.TypeOf(replyChan).Elem().Elem()).Interface().(proto.Response)
if err != nil {
reply.Header().SetGoError(err)
} else {
store.ExecuteCmd(method, args, reply)
if err := reply.Verify(args); err != nil {
reply.Header().SetGoError(err)
}
}
reflect.ValueOf(replyChan).Send(reflect.ValueOf(reply))
}
// compareStoreStatus ensures that the actual store status for the passed in
// store is updated correctly. It checks that the Desc.StoreID, Desc.Attrs,
// Desc.Node, Desc.Capacity.Capacity, NodeID, RangeCount, ReplicatedRangeCount
// are exactly correct and that the bytes and counts for Live, Key and Val are
// at least the expected value.
// The latest actual stats are returned.
func compareStoreStatus(t *testing.T, store *storage.Store, expectedStoreStatus *storage.StoreStatus, testNumber int) *storage.StoreStatus {
storeStatusKey := keys.StoreStatusKey(int32(store.Ident.StoreID))
gArgs, gReply := getArgs(storeStatusKey, 1, store.Ident.StoreID)
if err := store.ExecuteCmd(context.Background(), proto.Call{Args: gArgs, Reply: gReply}); err != nil {
t.Fatalf("%v: failure getting store status: %s", testNumber, err)
}
if gReply.Value == nil {
t.Errorf("%v: could not find store status at: %s", testNumber, storeStatusKey)
}
storeStatus := &storage.StoreStatus{}
if err := gogoproto.Unmarshal(gReply.Value.GetBytes(), storeStatus); err != nil {
t.Fatalf("%v: could not unmarshal store status: %+v", testNumber, gReply)
}
// Values much match exactly.
if expectedStoreStatus.Desc.StoreID != storeStatus.Desc.StoreID {
t.Errorf("%v: actual Desc.StoreID does not match expected\nexpected: %+v\nactual: %v\n", testNumber, expectedStoreStatus, storeStatus)
}
if !reflect.DeepEqual(expectedStoreStatus.Desc.Attrs, storeStatus.Desc.Attrs) {
t.Errorf("%v: actual Desc.Attrs does not match expected\nexpected: %+v\nactual: %v\n", testNumber, expectedStoreStatus, storeStatus)
}
if !reflect.DeepEqual(expectedStoreStatus.Desc.Node, storeStatus.Desc.Node) {
t.Errorf("%v: actual Desc.Attrs does not match expected\nexpected: %+v\nactual: %v\n", testNumber, expectedStoreStatus, storeStatus)
}
if storeStatus.Desc.Capacity.Capacity != expectedStoreStatus.Desc.Capacity.Capacity {
t.Errorf("%v: actual Desc.Capacity.Capacity does not match expected\nexpected: %+v\nactual: %v\n", testNumber, expectedStoreStatus, storeStatus)
}
if expectedStoreStatus.NodeID != storeStatus.NodeID {
t.Errorf("%v: actual node ID does not match expected\nexpected: %+v\nactual: %v\n", testNumber, expectedStoreStatus, storeStatus)
}
if expectedStoreStatus.RangeCount != storeStatus.RangeCount {
t.Errorf("%v: actual RangeCount does not match expected\nexpected: %+v\nactual: %v\n", testNumber, expectedStoreStatus, storeStatus)
}
if expectedStoreStatus.ReplicatedRangeCount != storeStatus.ReplicatedRangeCount {
t.Errorf("%v: actual ReplicatedRangeCount does not match expected\nexpected: %+v\nactual: %v\n", testNumber, expectedStoreStatus, storeStatus)
}
// Values should be >= to expected values.
if storeStatus.Stats.LiveBytes < expectedStoreStatus.Stats.LiveBytes {
t.Errorf("%v: actual Live Bytes is not greater or equal to expected\nexpected: %+v\nactual: %v\n", testNumber, expectedStoreStatus, storeStatus)
}
if storeStatus.Stats.KeyBytes < expectedStoreStatus.Stats.KeyBytes {
t.Errorf("%v: actual Key Bytes is not greater or equal to expected\nexpected: %+v\nactual: %v\n", testNumber, expectedStoreStatus, storeStatus)
}
if storeStatus.Stats.ValBytes < expectedStoreStatus.Stats.ValBytes {
t.Errorf("%v: actual Val Bytes is not greater or equal to expected\nexpected: %+v\nactual: %v\n", testNumber, expectedStoreStatus, storeStatus)
}
if storeStatus.Stats.LiveCount < expectedStoreStatus.Stats.LiveCount {
t.Errorf("%v: actual Live Count is not greater or equal to expected\nexpected: %+v\nactual: %v\n", testNumber, expectedStoreStatus, storeStatus)
}
if storeStatus.Stats.KeyCount < expectedStoreStatus.Stats.KeyCount {
t.Errorf("%v: actual Key Count is not greater or equal to expected\nexpected: %+v\nactual: %v\n", testNumber, expectedStoreStatus, storeStatus)
}
if storeStatus.Stats.ValCount < expectedStoreStatus.Stats.ValCount {
t.Errorf("%v: actual Val Count is not greater or equal to expected\nexpected: %+v\nactual: %v\n", testNumber, expectedStoreStatus, storeStatus)
}
return storeStatus
}
// SendBatch implements batch.Sender.
func (ls *LocalSender) SendBatch(ctx context.Context, ba proto.BatchRequest) (*proto.BatchResponse, error) {
trace := tracer.FromCtx(ctx)
var store *storage.Store
var err error
// If we aren't given a Replica, then a little bending over
// backwards here. This case applies exclusively to unittests.
if ba.RangeID == 0 || ba.Replica.StoreID == 0 {
var repl *proto.Replica
var rangeID proto.RangeID
rangeID, repl, err = ls.lookupReplica(ba.Key, ba.EndKey)
if err == nil {
ba.RangeID = rangeID
ba.Replica = *repl
}
}
ctx = log.Add(ctx,
log.Method, ba.Method(), // TODO(tschottdorf): Method() always `Batch`.
log.Key, ba.Key,
log.RangeID, ba.RangeID)
if err == nil {
store, err = ls.GetStore(ba.Replica.StoreID)
}
var br *proto.BatchResponse
if err == nil {
// For calls that read data within a txn, we can avoid uncertainty
// related retries in certain situations. If the node is in
// "CertainNodes", we need not worry about uncertain reads any
// more. Setting MaxTimestamp=Timestamp for the operation
// accomplishes that. See proto.Transaction.CertainNodes for details.
if ba.Txn != nil && ba.Txn.CertainNodes.Contains(ba.Replica.NodeID) {
// MaxTimestamp = Timestamp corresponds to no clock uncertainty.
trace.Event("read has no clock uncertainty")
ba.Txn.MaxTimestamp = ba.Txn.Timestamp
}
{
var tmpR proto.Response
// TODO(tschottdorf): &ba -> ba
tmpR, err = store.ExecuteCmd(ctx, &ba)
// TODO(tschottdorf): remove this dance once BatchResponse is returned.
if tmpR != nil {
br = tmpR.(*proto.BatchResponse)
if br.Error != nil {
panic(proto.ErrorUnexpectedlySet)
}
}
}
}
// TODO(tschottdorf): Later error needs to be associated to an index
// and ideally individual requests don't even have an error in their
// header. See #1891.
return br, err
}
// Send implements the client.Sender interface. The store is looked
// up from the store map if specified by header.Replica; otherwise,
// the command is being executed locally, and the replica is
// determined via lookup through each store's LookupRange method.
func (ls *LocalSender) Send(ctx context.Context, call proto.Call) {
var err error
var store *storage.Store
trace := tracer.FromCtx(ctx)
// If we aren't given a Replica, then a little bending over
// backwards here. This case applies exclusively to unittests.
header := call.Args.Header()
if header.RaftID == 0 || header.Replica.StoreID == 0 {
var repl *proto.Replica
var raftID proto.RaftID
raftID, repl, err = ls.lookupReplica(header.Key, header.EndKey)
if err == nil {
header.RaftID = raftID
header.Replica = *repl
}
}
ctx = log.Add(ctx,
log.Method, call.Method(),
log.Key, header.Key,
log.RaftID, header.RaftID)
if err == nil {
store, err = ls.GetStore(header.Replica.StoreID)
}
var reply proto.Response
if err == nil {
// For calls that read data within a txn, we can avoid uncertainty
// related retries in certain situations. If the node is in
// "CertainNodes", we need not worry about uncertain reads any
// more. Setting MaxTimestamp=Timestamp for the operation
// accomplishes that. See proto.Transaction.CertainNodes for details.
if header.Txn != nil && header.Txn.CertainNodes.Contains(header.Replica.NodeID) {
// MaxTimestamp = Timestamp corresponds to no clock uncertainty.
trace.Event("read has no clock uncertainty")
header.Txn.MaxTimestamp = header.Txn.Timestamp
}
reply, err = store.ExecuteCmd(ctx, call.Args)
}
if reply != nil {
gogoproto.Merge(call.Reply, reply)
}
if call.Reply.Header().Error != nil {
panic(proto.ErrorUnexpectedlySet)
}
if err != nil {
call.Reply.Header().SetGoError(err)
}
}
func createSplitRanges(store *storage.Store) (*proto.RangeDescriptor, *proto.RangeDescriptor, error) {
args := adminSplitArgs(proto.KeyMin, []byte("b"), 1, store.StoreID())
if _, err := store.ExecuteCmd(context.Background(), &args); err != nil {
return nil, nil, err
}
rangeA := store.LookupReplica([]byte("a"), nil)
rangeB := store.LookupReplica([]byte("c"), nil)
if bytes.Equal(rangeA.Desc().StartKey, rangeB.Desc().StartKey) {
log.Errorf("split ranges keys are equal %q!=%q", rangeA.Desc().StartKey, rangeB.Desc().StartKey)
}
return rangeA.Desc(), rangeB.Desc(), nil
}
func createSplitRanges(store *storage.Store) (*proto.RangeDescriptor, *proto.RangeDescriptor, error) {
args, reply := adminSplitArgs(engine.KeyMin, []byte("b"), 1, store.StoreID())
if err := store.ExecuteCmd(proto.AdminSplit, args, reply); err != nil {
return nil, nil, err
}
rangeA := store.LookupRange([]byte("a"), nil)
rangeB := store.LookupRange([]byte("c"), nil)
if bytes.Equal(rangeA.Desc().StartKey, rangeB.Desc().StartKey) {
log.Errorf("split ranges keys are equal %q!=%q", rangeA.Desc().StartKey, rangeB.Desc().StartKey)
}
return rangeA.Desc(), rangeB.Desc(), nil
}
// fillRange writes keys with the given prefix and associated values
// until bytes bytes have been written.
func fillRange(store *storage.Store, rangeID proto.RangeID, prefix proto.Key, bytes int64, t *testing.T) {
src := rand.New(rand.NewSource(0))
for {
var ms engine.MVCCStats
if err := engine.MVCCGetRangeStats(store.Engine(), rangeID, &ms); err != nil {
t.Fatal(err)
}
keyBytes, valBytes := ms.KeyBytes, ms.ValBytes
if keyBytes+valBytes >= bytes {
return
}
key := append(append([]byte(nil), prefix...), randutil.RandBytes(src, 100)...)
val := randutil.RandBytes(src, int(src.Int31n(1<<8)))
pArgs := putArgs(key, val, rangeID, store.StoreID())
pArgs.Timestamp = store.Clock().Now()
if _, err := store.ExecuteCmd(context.Background(), &pArgs); err != nil {
t.Fatal(err)
}
}
}