func TestCloneProto(t *testing.T) {
testCases := []struct {
pb proto.Message
shouldPanic bool
}{
// Uncloneable types (all contain UUID fields).
{&roachpb.StoreIdent{}, true},
{&enginepb.TxnMeta{}, true},
{&roachpb.Transaction{}, true},
{&roachpb.Error{}, true},
// Cloneable types. This includes all types for which a
// protoutil.Clone call exists in the codebase as of 2016-11-21.
{&config.ZoneConfig{}, false},
{&gossip.Info{}, false},
{&gossip.BootstrapInfo{}, false},
{&tracing.SpanContextCarrier{}, false},
{&sqlbase.IndexDescriptor{}, false},
{&roachpb.SplitTrigger{}, false},
{&roachpb.Value{}, false},
{&storagebase.ReplicaState{}, false},
{&roachpb.RangeDescriptor{}, false},
}
for _, tc := range testCases {
var clone proto.Message
var panicObj interface{}
func() {
defer func() {
panicObj = recover()
}()
clone = protoutil.Clone(tc.pb)
}()
if tc.shouldPanic {
if panicObj == nil {
t.Errorf("%T: expected panic but didn't get one", tc.pb)
} else {
if panicStr := fmt.Sprint(panicObj); !strings.Contains(panicStr, "attempt to clone") {
t.Errorf("%T: got unexpected panic %s", tc.pb, panicStr)
}
}
} else {
if panicObj != nil {
t.Errorf("%T: got unexpected panic %v", tc.pb, panicObj)
}
}
if panicObj == nil {
realClone := proto.Clone(tc.pb)
if !reflect.DeepEqual(clone, realClone) {
t.Errorf("%T: clone did not equal original. expected:\n%+v\ngot:\n%+v", tc.pb, realClone, clone)
}
}
}
}
func (ls *Stores) updateBootstrapInfo(bi *gossip.BootstrapInfo) error {
if bi.Timestamp.Less(ls.biLatestTS) {
return nil
}
ctx := ls.AnnotateCtx(context.TODO())
// Update the latest timestamp and set cached version.
ls.biLatestTS = bi.Timestamp
ls.latestBI = protoutil.Clone(bi).(*gossip.BootstrapInfo)
// Update all stores.
for _, s := range ls.storeMap {
if err := engine.MVCCPutProto(ctx, s.engine, nil, keys.StoreGossipKey(), hlc.ZeroTimestamp, nil, bi); err != nil {
return err
}
}
return nil
}
// loadState loads a ReplicaState from disk. The exception is the Desc field,
// which is updated transactionally, and is populated from the supplied
// RangeDescriptor under the convention that that is the latest committed
// version.
func loadState(
ctx context.Context, reader engine.Reader, desc *roachpb.RangeDescriptor,
) (storagebase.ReplicaState, error) {
var s storagebase.ReplicaState
// TODO(tschottdorf): figure out whether this is always synchronous with
// on-disk state (likely iffy during Split/ChangeReplica triggers).
s.Desc = protoutil.Clone(desc).(*roachpb.RangeDescriptor)
// Read the range lease.
lease, err := loadLease(ctx, reader, desc.RangeID)
if err != nil {
return storagebase.ReplicaState{}, err
}
s.Lease = &lease
if s.Frozen, err = loadFrozenStatus(ctx, reader, desc.RangeID); err != nil {
return storagebase.ReplicaState{}, err
}
if s.GCThreshold, err = loadGCThreshold(ctx, reader, desc.RangeID); err != nil {
return storagebase.ReplicaState{}, err
}
if s.TxnSpanGCThreshold, err = loadTxnSpanGCThreshold(ctx, reader, desc.RangeID); err != nil {
return storagebase.ReplicaState{}, err
}
if s.RaftAppliedIndex, s.LeaseAppliedIndex, err = loadAppliedIndex(
ctx, reader, desc.RangeID,
); err != nil {
return storagebase.ReplicaState{}, err
}
if s.Stats, err = loadMVCCStats(ctx, reader, desc.RangeID); err != nil {
return storagebase.ReplicaState{}, err
}
// The truncated state should not be optional (i.e. the pointer is
// pointless), but it is and the migration is not worth it.
truncState, err := loadTruncatedState(ctx, reader, desc.RangeID)
if err != nil {
return storagebase.ReplicaState{}, err
}
s.TruncatedState = &truncState
return s, nil
}
func convertBackfillError(tableDesc *sqlbase.TableDescriptor, b *client.Batch) error {
// A backfill on a new schema element has failed and the batch contains
// information useful in printing a sensible error. However
// convertBatchError() will only work correctly if the schema elements
// are "live" in the tableDesc.
desc := protoutil.Clone(tableDesc).(*sqlbase.TableDescriptor)
mutationID := desc.Mutations[0].MutationID
for _, mutation := range desc.Mutations {
if mutation.MutationID != mutationID {
// Mutations are applied in a FIFO order. Only apply the first set
// of mutations if they have the mutation ID we're looking for.
break
}
desc.MakeMutationComplete(mutation)
}
return convertBatchError(desc, b)
}
func TestCloneProto(t *testing.T) {
u := uuid.MakeV4()
testCases := []struct {
pb proto.Message
shouldPanic bool
}{
{&roachpb.StoreIdent{}, false},
{&roachpb.StoreIdent{ClusterID: uuid.MakeV4()}, true},
{&enginepb.TxnMeta{}, false},
{&enginepb.TxnMeta{ID: &u}, true},
{&roachpb.Transaction{}, false},
{&config.ZoneConfig{RangeMinBytes: 123, RangeMaxBytes: 456}, false},
}
for _, tc := range testCases {
var clone proto.Message
var panicObj interface{}
func() {
defer func() {
panicObj = recover()
}()
clone = protoutil.Clone(tc.pb)
}()
if tc.shouldPanic {
if panicObj == nil {
t.Errorf("%T: expected panic but didn't get one", tc.pb)
}
} else {
if panicObj != nil {
if panicStr := fmt.Sprint(panicObj); !strings.Contains(panicStr, "attempt to clone") {
t.Errorf("%T: got unexpected panic %s", tc.pb, panicStr)
}
}
}
if panicObj == nil {
realClone := proto.Clone(tc.pb)
if !reflect.DeepEqual(clone, realClone) {
t.Errorf("%T: clone did not equal original. expected:\n%+v\ngot:\n%+v", tc.pb, realClone, clone)
}
}
}
}
// InitOrJoinRequest executes a RequestLease command asynchronously and returns a
// channel on which the result will be posted. If there's already a request in
// progress, we join in waiting for the results of that request.
// It is an error to call InitOrJoinRequest() while a request is in progress
// naming another replica as lease holder.
//
// replica is used to schedule and execute async work (proposing a RequestLease
// command). replica.mu is locked when delivering results, so calls from the
// replica happen either before or after a result for a pending request has
// happened.
//
// transfer needs to be set if the request represents a lease transfer (as
// opposed to an extension, or acquiring the lease when none is held).
//
// Note: Once this function gets a context to be used for cancellation, instead
// of replica.store.Stopper().ShouldQuiesce(), care will be needed for cancelling
// the Raft command, similar to replica.addWriteCmd.
func (p *pendingLeaseRequest) InitOrJoinRequest(
replica *Replica,
nextLeaseHolder roachpb.ReplicaDescriptor,
timestamp hlc.Timestamp,
startKey roachpb.Key,
transfer bool,
) <-chan *roachpb.Error {
if nextLease, ok := p.RequestPending(); ok {
if nextLease.Replica.ReplicaID == nextLeaseHolder.ReplicaID {
// Join a pending request asking for the same replica to become lease
// holder.
return p.JoinRequest()
}
llChan := make(chan *roachpb.Error, 1)
// We can't join the request in progress.
llChan <- roachpb.NewErrorf("request for different replica in progress "+
"(requesting: %+v, in progress: %+v)",
nextLeaseHolder.ReplicaID, nextLease.Replica.ReplicaID)
return llChan
}
llChan := make(chan *roachpb.Error, 1)
// No request in progress. Let's propose a Lease command asynchronously.
// TODO(tschottdorf): get duration from configuration, either as a
// config flag or, later, dynamically adjusted.
startStasis := timestamp.Add(int64(replica.store.cfg.RangeLeaseActiveDuration), 0)
expiration := startStasis.Add(int64(replica.store.Clock().MaxOffset()), 0)
reqSpan := roachpb.Span{
Key: startKey,
}
var leaseReq roachpb.Request
reqLease := roachpb.Lease{
Start: timestamp,
StartStasis: startStasis,
Expiration: expiration,
Replica: nextLeaseHolder,
}
if transfer {
leaseReq = &roachpb.TransferLeaseRequest{
Span: reqSpan,
Lease: reqLease,
}
} else {
leaseReq = &roachpb.RequestLeaseRequest{
Span: reqSpan,
Lease: reqLease,
}
}
if replica.store.Stopper().RunAsyncTask(context.TODO(), func(ctx context.Context) {
ctx = replica.AnnotateCtx(ctx)
// Propose a RequestLease command and wait for it to apply.
ba := roachpb.BatchRequest{}
ba.Timestamp = replica.store.Clock().Now()
ba.RangeID = replica.RangeID
ba.Add(leaseReq)
if log.V(2) {
log.Infof(ctx, "sending lease request %v", leaseReq)
}
_, pErr := replica.Send(ctx, ba)
// Send result of lease to all waiter channels.
replica.mu.Lock()
defer replica.mu.Unlock()
for i, llChan := range p.llChans {
// Don't send the same pErr object twice; this can lead to races. We could
// clone every time but it's more efficient to send pErr itself to one of
// the channels (the last one; if we send it earlier the race can still
// happen).
if i == len(p.llChans)-1 {
llChan <- pErr
} else {
llChan <- protoutil.Clone(pErr).(*roachpb.Error) // works with `nil`
}
}
p.llChans = p.llChans[:0]
p.nextLease = roachpb.Lease{}
}) != nil {
// We failed to start the asynchronous task. Send a blank NotLeaseHolderError
// back to indicate that we have no idea who the range lease holder might
// be; we've withdrawn from active duty.
llChan <- roachpb.NewError(
newNotLeaseHolderError(nil, replica.store.StoreID(), replica.mu.state.Desc))
return llChan
}
p.llChans = append(p.llChans, llChan)
p.nextLease = reqLease
return llChan
}
func TestSchemaChangeProcess(t *testing.T) {
defer leaktest.AfterTest(t)()
// The descriptor changes made must have an immediate effect
// so disable leases on tables.
defer csql.TestDisableTableLeases()()
params, _ := createTestServerParams()
// Disable external processing of mutations.
params.Knobs.SQLSchemaChanger = &csql.SchemaChangerTestingKnobs{
AsyncExecNotification: asyncSchemaChangerDisabled,
}
s, sqlDB, kvDB := serverutils.StartServer(t, params)
defer s.Stopper().Stop()
var id = sqlbase.ID(keys.MaxReservedDescID + 2)
var node = roachpb.NodeID(2)
stopper := stop.NewStopper()
leaseMgr := csql.NewLeaseManager(
&base.NodeIDContainer{},
*kvDB,
hlc.NewClock(hlc.UnixNano, time.Nanosecond),
csql.LeaseManagerTestingKnobs{},
stopper,
&csql.MemoryMetrics{},
)
defer stopper.Stop()
changer := csql.NewSchemaChangerForTesting(id, 0, node, *kvDB, leaseMgr)
if _, err := sqlDB.Exec(`
CREATE DATABASE t;
CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, INDEX foo(v));
INSERT INTO t.test VALUES ('a', 'b'), ('c', 'd');
`); err != nil {
t.Fatal(err)
}
// Read table descriptor for version.
tableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test")
expectedVersion := tableDesc.Version
desc, err := changer.MaybeIncrementVersion()
if err != nil {
t.Fatal(err)
}
tableDesc = desc.GetTable()
newVersion := tableDesc.Version
if newVersion != expectedVersion {
t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion)
}
isDone, err := changer.IsDone()
if err != nil {
t.Fatal(err)
}
if !isDone {
t.Fatalf("table expected to not have an outstanding schema change: %v", tableDesc)
}
// Check that MaybeIncrementVersion increments the version
// correctly.
expectedVersion++
tableDesc.UpVersion = true
if err := kvDB.Put(
context.TODO(),
sqlbase.MakeDescMetadataKey(tableDesc.ID),
sqlbase.WrapDescriptor(tableDesc),
); err != nil {
t.Fatal(err)
}
isDone, err = changer.IsDone()
if err != nil {
t.Fatal(err)
}
if isDone {
t.Fatalf("table expected to have an outstanding schema change: %v", desc.GetTable())
}
desc, err = changer.MaybeIncrementVersion()
if err != nil {
t.Fatal(err)
}
tableDesc = desc.GetTable()
savedTableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test")
newVersion = tableDesc.Version
if newVersion != expectedVersion {
t.Fatalf("bad version in returned desc; e = %d, v = %d", expectedVersion, newVersion)
}
newVersion = savedTableDesc.Version
if newVersion != expectedVersion {
t.Fatalf("bad version in saved desc; e = %d, v = %d", expectedVersion, newVersion)
}
isDone, err = changer.IsDone()
if err != nil {
t.Fatal(err)
}
if !isDone {
t.Fatalf("table expected to not have an outstanding schema change: %v", tableDesc)
}
// Check that RunStateMachineBeforeBackfill doesn't do anything
// if there are no mutations queued.
if err := changer.RunStateMachineBeforeBackfill(); err != nil {
t.Fatal(err)
}
tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test")
newVersion = tableDesc.Version
if newVersion != expectedVersion {
t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion)
}
// Check that RunStateMachineBeforeBackfill functions properly.
expectedVersion = tableDesc.Version
// Make a copy of the index for use in a mutation.
index := protoutil.Clone(&tableDesc.Indexes[0]).(*sqlbase.IndexDescriptor)
index.Name = "bar"
index.ID = tableDesc.NextIndexID
tableDesc.NextIndexID++
changer = csql.NewSchemaChangerForTesting(id, tableDesc.NextMutationID, node, *kvDB, leaseMgr)
tableDesc.Mutations = append(tableDesc.Mutations, sqlbase.DescriptorMutation{
Descriptor_: &sqlbase.DescriptorMutation_Index{Index: index},
Direction: sqlbase.DescriptorMutation_ADD,
State: sqlbase.DescriptorMutation_DELETE_ONLY,
MutationID: tableDesc.NextMutationID,
})
tableDesc.NextMutationID++
// Run state machine in both directions.
for _, direction := range []sqlbase.DescriptorMutation_Direction{sqlbase.DescriptorMutation_ADD, sqlbase.DescriptorMutation_DROP} {
tableDesc.Mutations[0].Direction = direction
expectedVersion++
if err := kvDB.Put(
context.TODO(),
sqlbase.MakeDescMetadataKey(tableDesc.ID),
sqlbase.WrapDescriptor(tableDesc),
); err != nil {
t.Fatal(err)
}
// The expected end state.
expectedState := sqlbase.DescriptorMutation_WRITE_ONLY
if direction == sqlbase.DescriptorMutation_DROP {
expectedState = sqlbase.DescriptorMutation_DELETE_ONLY
}
// Run two times to ensure idempotency of operations.
for i := 0; i < 2; i++ {
if err := changer.RunStateMachineBeforeBackfill(); err != nil {
t.Fatal(err)
}
tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test")
newVersion = tableDesc.Version
if newVersion != expectedVersion {
t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion)
}
state := tableDesc.Mutations[0].State
if state != expectedState {
t.Fatalf("bad state; e = %d, v = %d", expectedState, state)
}
}
}
// RunStateMachineBeforeBackfill() doesn't complete the schema change.
isDone, err = changer.IsDone()
if err != nil {
t.Fatal(err)
}
if isDone {
t.Fatalf("table expected to have an outstanding schema change: %v", tableDesc)
}
}
// Send implements the batch.Sender interface. If the request is part of a
// transaction, the TxnCoordSender adds the transaction to a map of active
// transactions and begins heartbeating it. Every subsequent request for the
// same transaction updates the lastUpdate timestamp to prevent live
// transactions from being considered abandoned and garbage collected.
// Read/write mutating requests have their key or key range added to the
// transaction's interval tree of key ranges for eventual cleanup via resolved
// write intents; they're tagged to an outgoing EndTransaction request, with
// the receiving replica in charge of resolving them.
func (tc *TxnCoordSender) Send(
ctx context.Context, ba roachpb.BatchRequest,
) (*roachpb.BatchResponse, *roachpb.Error) {
// Start new or pick up active trace. From here on, there's always an active
// Trace, though its overhead is small unless it's sampled.
sp := opentracing.SpanFromContext(ctx)
var tracer opentracing.Tracer
if sp == nil {
tracer = tc.AmbientContext.Tracer
sp = tracer.StartSpan(opTxnCoordSender)
defer sp.Finish()
ctx = opentracing.ContextWithSpan(ctx, sp)
} else {
tracer = sp.Tracer()
}
startNS := tc.clock.PhysicalNow()
if ba.Txn != nil {
// If this request is part of a transaction...
if err := tc.maybeBeginTxn(&ba); err != nil {
return nil, roachpb.NewError(err)
}
txnID := *ba.Txn.ID
// Associate the txnID with the trace. We need to do this after the
// maybeBeginTxn call. We set both a baggage item and a tag because only
// tags show up in the LIghtstep UI.
txnIDStr := txnID.String()
sp.SetTag("txnID", txnIDStr)
sp.SetBaggageItem("txnID", txnIDStr)
var et *roachpb.EndTransactionRequest
var hasET bool
{
var rArgs roachpb.Request
rArgs, hasET = ba.GetArg(roachpb.EndTransaction)
if hasET {
et = rArgs.(*roachpb.EndTransactionRequest)
if len(et.Key) != 0 {
return nil, roachpb.NewErrorf("EndTransaction must not have a Key set")
}
et.Key = ba.Txn.Key
if len(et.IntentSpans) > 0 {
// TODO(tschottdorf): it may be useful to allow this later.
// That would be part of a possible plan to allow txns which
// write on multiple coordinators.
return nil, roachpb.NewErrorf("client must not pass intents to EndTransaction")
}
}
}
if pErr := func() *roachpb.Error {
tc.Lock()
defer tc.Unlock()
if pErr := tc.maybeRejectClientLocked(ctx, *ba.Txn); pErr != nil {
return pErr
}
if !hasET {
return nil
}
// Everything below is carried out only when trying to commit.
// Populate et.IntentSpans, taking into account both any existing
// and new writes, and taking care to perform proper deduplication.
txnMeta := tc.txns[txnID]
distinctSpans := true
if txnMeta != nil {
et.IntentSpans = txnMeta.keys
// Defensively set distinctSpans to false if we had any previous
// requests in this transaction. This effectively limits the distinct
// spans optimization to 1pc transactions.
distinctSpans = len(txnMeta.keys) == 0
}
// We can't pass in a batch response here to better limit the key
// spans as we don't know what is going to be affected. This will
// affect queries such as `DELETE FROM my.table LIMIT 10` when
// executed as a 1PC transaction. e.g.: a (BeginTransaction,
// DeleteRange, EndTransaction) batch.
ba.IntentSpanIterate(nil, func(key, endKey roachpb.Key) {
et.IntentSpans = append(et.IntentSpans, roachpb.Span{
Key: key,
EndKey: endKey,
})
})
// TODO(peter): Populate DistinctSpans on all batches, not just batches
// which contain an EndTransactionRequest.
var distinct bool
// The request might already be used by an outgoing goroutine, so
// we can't safely mutate anything in-place (as MergeSpans does).
et.IntentSpans = append([]roachpb.Span(nil), et.IntentSpans...)
et.IntentSpans, distinct = roachpb.MergeSpans(et.IntentSpans)
ba.Header.DistinctSpans = distinct && distinctSpans
if len(et.IntentSpans) == 0 {
// If there aren't any intents, then there's factually no
// transaction to end. Read-only txns have all of their state
// in the client.
return roachpb.NewErrorf("cannot commit a read-only transaction")
}
if txnMeta != nil {
txnMeta.keys = et.IntentSpans
}
return nil
}(); pErr != nil {
return nil, pErr
}
if hasET && log.V(1) {
for _, intent := range et.IntentSpans {
log.Eventf(ctx, "intent: [%s,%s)", intent.Key, intent.EndKey)
}
}
}
// Embed the trace metadata into the header for use by RPC recipients. We need
// to do this after the maybeBeginTxn call above.
// TODO(tschottdorf): To get rid of the spurious alloc below we need to
// implement the carrier interface on ba.Header or make Span non-nullable,
// both of which force all of ba on the Heap. It's already there, so may
// not be a big deal, but ba should live on the stack. Also not easy to use
// a buffer pool here since anything that goes into the RPC layer could be
// used by goroutines we didn't wait for.
if ba.TraceContext == nil {
ba.TraceContext = &tracing.SpanContextCarrier{}
} else {
// We didn't make this object but are about to mutate it, so we
// have to take a copy - the original might already have been
// passed to the RPC layer.
ba.TraceContext = protoutil.Clone(ba.TraceContext).(*tracing.SpanContextCarrier)
}
if err := tracer.Inject(sp.Context(), basictracer.Delegator, ba.TraceContext); err != nil {
return nil, roachpb.NewError(err)
}
// Send the command through wrapped sender, taking appropriate measures
// on error.
var br *roachpb.BatchResponse
{
var pErr *roachpb.Error
br, pErr = tc.wrapped.Send(ctx, ba)
if _, ok := pErr.GetDetail().(*roachpb.OpRequiresTxnError); ok {
// TODO(tschottdorf): needs to keep the trace.
br, pErr = tc.resendWithTxn(ba)
}
if pErr = tc.updateState(ctx, startNS, ba, br, pErr); pErr != nil {
log.Eventf(ctx, "error: %s", pErr)
return nil, pErr
}
}
if br.Txn == nil {
return br, nil
}
if _, ok := ba.GetArg(roachpb.EndTransaction); !ok {
return br, nil
}
// If the --linearizable flag is set, we want to make sure that
// all the clocks in the system are past the commit timestamp
// of the transaction. This is guaranteed if either
// - the commit timestamp is MaxOffset behind startNS
// - MaxOffset ns were spent in this function
// when returning to the client. Below we choose the option
// that involves less waiting, which is likely the first one
// unless a transaction commits with an odd timestamp.
if tsNS := br.Txn.Timestamp.WallTime; startNS > tsNS {
startNS = tsNS
}
sleepNS := tc.clock.MaxOffset() -
time.Duration(tc.clock.PhysicalNow()-startNS)
if tc.linearizable && sleepNS > 0 {
defer func() {
if log.V(1) {
log.Infof(ctx, "%v: waiting %s on EndTransaction for linearizability", br.Txn.Short(), util.TruncateDuration(sleepNS, time.Millisecond))
}
time.Sleep(sleepNS)
}()
}
if br.Txn.Status != roachpb.PENDING {
tc.Lock()
tc.cleanupTxnLocked(ctx, *br.Txn)
tc.Unlock()
}
return br, nil
}