func (ls *Stores) updateBootstrapInfo(bi *gossip.BootstrapInfo) error { if bi.Timestamp.Less(ls.biLatestTS) { return nil } // Update the latest timestamp and set cached version. ls.biLatestTS = bi.Timestamp ls.latestBI = protoutil.Clone(bi).(*gossip.BootstrapInfo) // Update all stores. for _, s := range ls.storeMap { if err := engine.MVCCPutProto(context.Background(), s.engine, nil, keys.StoreGossipKey(), hlc.ZeroTimestamp, nil, bi); err != nil { return err } } return nil }
// loadState loads a ReplicaState from disk. The exception is the Desc field, // which is updated transactionally, and is populated from the supplied // RangeDescriptor under the convention that that is the latest committed // version. func loadState( ctx context.Context, reader engine.Reader, desc *roachpb.RangeDescriptor, ) (storagebase.ReplicaState, error) { var s storagebase.ReplicaState // TODO(tschottdorf): figure out whether this is always synchronous with // on-disk state (likely iffy during Split/ChangeReplica triggers). s.Desc = protoutil.Clone(desc).(*roachpb.RangeDescriptor) // Read the range lease. var err error if s.Lease, err = loadLease(ctx, reader, desc.RangeID); err != nil { return storagebase.ReplicaState{}, err } if s.Frozen, err = loadFrozenStatus(ctx, reader, desc.RangeID); err != nil { return storagebase.ReplicaState{}, err } if s.GCThreshold, err = loadGCThreshold(ctx, reader, desc.RangeID); err != nil { return storagebase.ReplicaState{}, err } if s.RaftAppliedIndex, s.LeaseAppliedIndex, err = loadAppliedIndex( ctx, reader, desc.RangeID, ); err != nil { return storagebase.ReplicaState{}, err } if s.Stats, err = loadMVCCStats(ctx, reader, desc.RangeID); err != nil { return storagebase.ReplicaState{}, err } // The truncated state should not be optional (i.e. the pointer is // pointless), but it is and the migration is not worth it. truncState, err := loadTruncatedState(ctx, reader, desc.RangeID) if err != nil { return storagebase.ReplicaState{}, err } s.TruncatedState = &truncState return s, nil }
func TestCloneProto(t *testing.T) { testCases := []struct { pb proto.Message shouldPanic bool }{ {&roachpb.StoreIdent{}, false}, {&roachpb.StoreIdent{ClusterID: uuid.MakeV4()}, true}, {&roachpb.TxnMeta{}, false}, {&roachpb.TxnMeta{ID: uuid.NewV4()}, true}, {&roachpb.Transaction{}, false}, {&config.ZoneConfig{RangeMinBytes: 123, RangeMaxBytes: 456}, false}, } for _, tc := range testCases { var clone proto.Message var panicObj interface{} func() { defer func() { panicObj = recover() }() clone = protoutil.Clone(tc.pb) }() if tc.shouldPanic { if panicObj == nil { t.Errorf("%T: expected panic but didn't get one", tc.pb) } } else { if panicObj != nil { if panicStr := fmt.Sprint(panicObj); !strings.Contains(panicStr, "attempt to clone") { t.Errorf("%T: got unexpected panic %s", tc.pb, panicStr) } } } if panicObj == nil { realClone := proto.Clone(tc.pb) if !reflect.DeepEqual(clone, realClone) { t.Errorf("%T: clone did not equal original. expected:\n%+v\ngot:\n%+v", tc.pb, realClone, clone) } } } }
// Send implements the batch.Sender interface. If the request is part of a // transaction, the TxnCoordSender adds the transaction to a map of active // transactions and begins heartbeating it. Every subsequent request for the // same transaction updates the lastUpdate timestamp to prevent live // transactions from being considered abandoned and garbage collected. // Read/write mutating requests have their key or key range added to the // transaction's interval tree of key ranges for eventual cleanup via resolved // write intents; they're tagged to an outgoing EndTransaction request, with // the receiving replica in charge of resolving them. func (tc *TxnCoordSender) Send(ctx context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) { { // Start new or pick up active trace and embed its trace metadata into // header for use by RPC recipients. From here on, there's always an active // Trace, though its overhead is small unless it's sampled. sp := opentracing.SpanFromContext(ctx) // TODO(radu): once contexts are plumbed correctly, we should use the Tracer // from ctx. tracer := tracing.TracerFromCtx(tc.ctx) if sp == nil { sp = tracer.StartSpan(opTxnCoordSender) defer sp.Finish() ctx = opentracing.ContextWithSpan(ctx, sp) } // TODO(tschottdorf): To get rid of the spurious alloc below we need to // implement the carrier interface on ba.Header or make Span non-nullable, // both of which force all of ba on the Heap. It's already there, so may // not be a big deal, but ba should live on the stack. Also not easy to use // a buffer pool here since anything that goes into the RPC layer could be // used by goroutines we didn't wait for. if ba.Header.Trace == nil { ba.Header.Trace = &tracing.Span{} } else { // We didn't make this object but are about to mutate it, so we // have to take a copy - the original might already have been // passed to the RPC layer. ba.Header.Trace = protoutil.Clone(ba.Header.Trace).(*tracing.Span) } if err := tracer.Inject(sp.Context(), basictracer.Delegator, ba.Trace); err != nil { return nil, roachpb.NewError(err) } } startNS := tc.clock.PhysicalNow() if ba.Txn != nil { // If this request is part of a transaction... if err := tc.maybeBeginTxn(&ba); err != nil { return nil, roachpb.NewError(err) } var et *roachpb.EndTransactionRequest var hasET bool { var rArgs roachpb.Request rArgs, hasET = ba.GetArg(roachpb.EndTransaction) if hasET { et = rArgs.(*roachpb.EndTransactionRequest) if len(et.Key) != 0 { return nil, roachpb.NewErrorf("EndTransaction must not have a Key set") } et.Key = ba.Txn.Key if len(et.IntentSpans) > 0 { // TODO(tschottdorf): it may be useful to allow this later. // That would be part of a possible plan to allow txns which // write on multiple coordinators. return nil, roachpb.NewErrorf("client must not pass intents to EndTransaction") } } } if pErr := func() *roachpb.Error { tc.Lock() defer tc.Unlock() if pErr := tc.maybeRejectClientLocked(ctx, *ba.Txn); pErr != nil { return pErr } if !hasET { return nil } // Everything below is carried out only when trying to commit. // Populate et.IntentSpans, taking into account both any existing // and new writes, and taking care to perform proper deduplication. txnMeta := tc.txns[*ba.Txn.ID] distinctSpans := true if txnMeta != nil { et.IntentSpans = txnMeta.keys // Defensively set distinctSpans to false if we had any previous // requests in this transaction. This effectively limits the distinct // spans optimization to 1pc transactions. distinctSpans = len(txnMeta.keys) == 0 } ba.IntentSpanIterate(func(key, endKey roachpb.Key) { et.IntentSpans = append(et.IntentSpans, roachpb.Span{ Key: key, EndKey: endKey, }) }) // TODO(peter): Populate DistinctSpans on all batches, not just batches // which contain an EndTransactionRequest. var distinct bool // The request might already be used by an outgoing goroutine, so // we can't safely mutate anything in-place (as MergeSpans does). et.IntentSpans = append([]roachpb.Span(nil), et.IntentSpans...) et.IntentSpans, distinct = roachpb.MergeSpans(et.IntentSpans) ba.Header.DistinctSpans = distinct && distinctSpans if len(et.IntentSpans) == 0 { // If there aren't any intents, then there's factually no // transaction to end. Read-only txns have all of their state // in the client. return roachpb.NewErrorf("cannot commit a read-only transaction") } if txnMeta != nil { txnMeta.keys = et.IntentSpans } return nil }(); pErr != nil { return nil, pErr } if hasET && log.V(1) { for _, intent := range et.IntentSpans { log.Tracef(ctx, "intent: [%s,%s)", intent.Key, intent.EndKey) } } } // Send the command through wrapped sender, taking appropriate measures // on error. var br *roachpb.BatchResponse { var pErr *roachpb.Error br, pErr = tc.wrapped.Send(ctx, ba) if _, ok := pErr.GetDetail().(*roachpb.OpRequiresTxnError); ok { // TODO(tschottdorf): needs to keep the trace. br, pErr = tc.resendWithTxn(ba) } if pErr = tc.updateState(startNS, ctx, ba, br, pErr); pErr != nil { log.Tracef(ctx, "error: %s", pErr) return nil, pErr } } if br.Txn == nil { return br, nil } if _, ok := ba.GetArg(roachpb.EndTransaction); !ok { return br, nil } // If the --linearizable flag is set, we want to make sure that // all the clocks in the system are past the commit timestamp // of the transaction. This is guaranteed if either // - the commit timestamp is MaxOffset behind startNS // - MaxOffset ns were spent in this function // when returning to the client. Below we choose the option // that involves less waiting, which is likely the first one // unless a transaction commits with an odd timestamp. if tsNS := br.Txn.Timestamp.WallTime; startNS > tsNS { startNS = tsNS } sleepNS := tc.clock.MaxOffset() - time.Duration(tc.clock.PhysicalNow()-startNS) if tc.linearizable && sleepNS > 0 { defer func() { if log.V(1) { log.Infof(ctx, "%v: waiting %s on EndTransaction for linearizability", br.Txn.ID.Short(), util.TruncateDuration(sleepNS, time.Millisecond)) } time.Sleep(sleepNS) }() } if br.Txn.Status != roachpb.PENDING { tc.Lock() tc.cleanupTxnLocked(ctx, *br.Txn) tc.Unlock() } return br, nil }
// InitOrJoinRequest executes a RequestLease command asynchronously and returns a // channel on which the result will be posted. If there's already a request in // progress, we join in waiting for the results of that request. // It is an error to call InitOrJoinRequest() while a request is in progress // naming another replica as lease holder. // // replica is used to schedule and execute async work (proposing a RequestLease // command). replica.mu is locked when delivering results, so calls from the // replica happen either before or after a result for a pending request has // happened. // // transfer needs to be set if the request represents a lease transfer (as // opposed to an extension, or acquiring the lease when none is held). // // Note: Once this function gets a context to be used for cancellation, instead // of replica.store.Stopper().ShouldQuiesce(), care will be needed for cancelling // the Raft command, similar to replica.addWriteCmd. func (p *pendingLeaseRequest) InitOrJoinRequest( replica *Replica, nextLeaseHolder roachpb.ReplicaDescriptor, timestamp hlc.Timestamp, startKey roachpb.Key, transfer bool, ) <-chan *roachpb.Error { if nextLease := p.RequestPending(); nextLease != nil { if nextLease.Replica.ReplicaID == nextLeaseHolder.ReplicaID { // Join a pending request asking for the same replica to become lease // holder. return p.JoinRequest() } llChan := make(chan *roachpb.Error, 1) // We can't join the request in progress. llChan <- roachpb.NewErrorf("request for different replica in progress "+ "(requesting: %+v, in progress: %+v)", nextLeaseHolder.ReplicaID, nextLease.Replica.ReplicaID) return llChan } llChan := make(chan *roachpb.Error, 1) // No request in progress. Let's propose a Lease command asynchronously. // TODO(tschottdorf): get duration from configuration, either as a // config flag or, later, dynamically adjusted. startStasis := timestamp.Add(int64(replica.store.ctx.rangeLeaseActiveDuration), 0) expiration := startStasis.Add(int64(replica.store.Clock().MaxOffset()), 0) reqSpan := roachpb.Span{ Key: startKey, } var leaseReq roachpb.Request reqLease := roachpb.Lease{ Start: timestamp, StartStasis: startStasis, Expiration: expiration, Replica: nextLeaseHolder, } if transfer { leaseReq = &roachpb.TransferLeaseRequest{ Span: reqSpan, Lease: reqLease, } } else { leaseReq = &roachpb.RequestLeaseRequest{ Span: reqSpan, Lease: reqLease, } } if replica.store.Stopper().RunAsyncTask(func() { // Propose a RequestLease command and wait for it to apply. var execPErr *roachpb.Error ba := roachpb.BatchRequest{} ba.Timestamp = replica.store.Clock().Now() ba.RangeID = replica.RangeID ba.Add(leaseReq) // Send lease request directly to raft in order to skip unnecessary // checks from normal request machinery, (e.g. the command queue). // Note that the command itself isn't traced, but usually the caller // waiting for the result has an active Trace. ch, _, err := replica.proposeRaftCommand( replica.context(context.Background()), ba) if err != nil { execPErr = roachpb.NewError(err) } else { // If the command was committed, wait for the range to apply it. select { case c := <-ch: if c.Err != nil { if log.V(1) { log.Infof("failed to acquire lease for replica %s: %s", replica.store, c.Err) } execPErr = c.Err } case <-replica.store.Stopper().ShouldQuiesce(): execPErr = roachpb.NewError( replica.newNotLeaseHolderError(nil, replica.store.StoreID(), replica.Desc())) } } // Send result of lease to all waiter channels. replica.mu.Lock() defer replica.mu.Unlock() for i, llChan := range p.llChans { // Don't send the same pErr object twice; this can lead to races. We could // clone every time but it's more efficient to send pErr itself to one of // the channels (the last one; if we send it earlier the race can still // happen). if i == len(p.llChans)-1 { llChan <- execPErr } else { llChan <- protoutil.Clone(execPErr).(*roachpb.Error) // works with `nil` } } p.llChans = p.llChans[:0] p.nextLease = roachpb.Lease{} }) != nil { // We failed to start the asynchronous task. Send a blank NotLeaseHolderError // back to indicate that we have no idea who the range lease holder might // be; we've withdrawn from active duty. llChan <- roachpb.NewError( replica.newNotLeaseHolderError(nil, replica.store.StoreID(), replica.mu.state.Desc)) return llChan } p.llChans = append(p.llChans, llChan) p.nextLease = reqLease return llChan }
func TestSchemaChangeProcess(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() params, _ := createTestServerParams() // Disable external processing of mutations. params.Knobs.SQLSchemaChangeManager = &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: schemaChangeManagerDisabled, } s, sqlDB, kvDB := serverutils.StartServer(t, params) defer s.Stopper().Stop() var id = sqlbase.ID(keys.MaxReservedDescID + 2) var node = roachpb.NodeID(2) stopper := stop.NewStopper() leaseMgr := csql.NewLeaseManager(0, *kvDB, hlc.NewClock(hlc.UnixNano), csql.LeaseManagerTestingKnobs{}, stopper) defer stopper.Stop() changer := csql.NewSchemaChangerForTesting(id, 0, node, *kvDB, leaseMgr) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, INDEX foo(v)); INSERT INTO t.test VALUES ('a', 'b'), ('c', 'd'); `); err != nil { t.Fatal(err) } // Read table descriptor for version. tableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test") expectedVersion := tableDesc.Version desc, err := changer.MaybeIncrementVersion() if err != nil { t.Fatal(err) } tableDesc = desc.GetTable() newVersion := tableDesc.Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } isDone, err := changer.IsDone() if err != nil { t.Fatal(err) } if !isDone { t.Fatalf("table expected to not have an outstanding schema change: %v", tableDesc) } // Check that MaybeIncrementVersion increments the version // correctly. expectedVersion++ tableDesc.UpVersion = true if err := kvDB.Put( sqlbase.MakeDescMetadataKey(tableDesc.ID), sqlbase.WrapDescriptor(tableDesc), ); err != nil { t.Fatal(err) } isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if isDone { t.Fatalf("table expected to have an outstanding schema change: %v", desc.GetTable()) } desc, err = changer.MaybeIncrementVersion() if err != nil { t.Fatal(err) } tableDesc = desc.GetTable() savedTableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test") newVersion = tableDesc.Version if newVersion != expectedVersion { t.Fatalf("bad version in returned desc; e = %d, v = %d", expectedVersion, newVersion) } newVersion = savedTableDesc.Version if newVersion != expectedVersion { t.Fatalf("bad version in saved desc; e = %d, v = %d", expectedVersion, newVersion) } isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if !isDone { t.Fatalf("table expected to not have an outstanding schema change: %v", tableDesc) } // Check that RunStateMachineBeforeBackfill doesn't do anything // if there are no mutations queued. if err := changer.RunStateMachineBeforeBackfill(); err != nil { t.Fatal(err) } tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test") newVersion = tableDesc.Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } // Check that RunStateMachineBeforeBackfill functions properly. expectedVersion = tableDesc.Version // Make a copy of the index for use in a mutation. index := protoutil.Clone(&tableDesc.Indexes[0]).(*sqlbase.IndexDescriptor) index.Name = "bar" index.ID = tableDesc.NextIndexID tableDesc.NextIndexID++ changer = csql.NewSchemaChangerForTesting(id, tableDesc.NextMutationID, node, *kvDB, leaseMgr) tableDesc.Mutations = append(tableDesc.Mutations, sqlbase.DescriptorMutation{ Descriptor_: &sqlbase.DescriptorMutation_Index{Index: index}, Direction: sqlbase.DescriptorMutation_ADD, State: sqlbase.DescriptorMutation_DELETE_ONLY, MutationID: tableDesc.NextMutationID, }) tableDesc.NextMutationID++ // Run state machine in both directions. for _, direction := range []sqlbase.DescriptorMutation_Direction{sqlbase.DescriptorMutation_ADD, sqlbase.DescriptorMutation_DROP} { tableDesc.Mutations[0].Direction = direction expectedVersion++ if err := kvDB.Put( sqlbase.MakeDescMetadataKey(tableDesc.ID), sqlbase.WrapDescriptor(tableDesc), ); err != nil { t.Fatal(err) } // The expected end state. expectedState := sqlbase.DescriptorMutation_WRITE_ONLY if direction == sqlbase.DescriptorMutation_DROP { expectedState = sqlbase.DescriptorMutation_DELETE_ONLY } // Run two times to ensure idempotency of operations. for i := 0; i < 2; i++ { if err := changer.RunStateMachineBeforeBackfill(); err != nil { t.Fatal(err) } tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test") newVersion = tableDesc.Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } state := tableDesc.Mutations[0].State if state != expectedState { t.Fatalf("bad state; e = %d, v = %d", expectedState, state) } } } // RunStateMachineBeforeBackfill() doesn't complete the schema change. isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if isDone { t.Fatalf("table expected to have an outstanding schema change: %v", tableDesc) } }
// TestStoreRangeSplitRaceUninitializedRHS reproduces #7600 (before it was // fixed). While splits are happening, we simulate incoming messages for the // right-hand side to trigger a race between the creation of the proper replica // and the uninitialized replica reacting to messages. func TestStoreRangeSplitRaceUninitializedRHS(t *testing.T) { defer leaktest.AfterTest(t)() mtc := &multiTestContext{} storeCtx := storage.TestStoreContext() // An aggressive tick interval lets groups communicate more and thus // triggers test failures much more reliably. We can't go too aggressive // or race tests never make any progress. storeCtx.RaftTickInterval = 50 * time.Millisecond storeCtx.RaftElectionTimeoutTicks = 2 currentTrigger := make(chan *roachpb.SplitTrigger) seen := make(map[storagebase.CmdIDKey]struct{}) storeCtx.TestingKnobs.TestingCommandFilter = func(args storagebase.FilterArgs) *roachpb.Error { et, ok := args.Req.(*roachpb.EndTransactionRequest) if !ok || et.InternalCommitTrigger == nil { return nil } trigger := protoutil.Clone(et.InternalCommitTrigger.GetSplitTrigger()).(*roachpb.SplitTrigger) if trigger != nil && len(trigger.NewDesc.Replicas) == 2 && args.Hdr.Txn.Epoch == 0 && args.Sid == mtc.stores[0].StoreID() { if _, ok := seen[args.CmdID]; ok { return nil } // Without replay protection, a single reproposal locks up the // test. seen[args.CmdID] = struct{}{} currentTrigger <- trigger return roachpb.NewError(roachpb.NewReadWithinUncertaintyIntervalError(args.Hdr.Timestamp, args.Hdr.Timestamp)) } return nil } mtc.storeContext = &storeCtx mtc.Start(t, 2) defer mtc.Stop() leftRange := mtc.stores[0].LookupReplica(roachpb.RKey("a"), nil) // We'll fake messages from term 1, ..., .magicIters-1. The exact number // doesn't matter for anything but for its likelihood of triggering the // race. const magicIters = 5 // Replicate the left range onto the second node. We don't wait since we // don't actually care what the second node does. All we want is that the // first node isn't surprised by messages from that node. mtc.replicateRange(leftRange.RangeID, 1) for i := 0; i < 10; i++ { var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() // Split the data range. The split keys are chosen so that they move // towards "a" (so that the range being split is always the first // range). splitKey := roachpb.Key(encoding.EncodeVarintDescending([]byte("a"), int64(i))) splitArgs := adminSplitArgs(keys.SystemMax, splitKey) if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &splitArgs); pErr != nil { t.Fatal(pErr) } }() go func() { defer wg.Done() trigger := <-currentTrigger // our own copy // Make sure the first node is first for convenience. replicas := trigger.NewDesc.Replicas if replicas[0].NodeID > replicas[1].NodeID { tmp := replicas[1] replicas[1] = replicas[0] replicas[0] = tmp } // Send a few vote requests which look like they're from the other // node's right hand side of the split. This triggers a race which // is discussed in #7600 (briefly, the creation of the right hand // side in the split trigger was racing with the uninitialized // version for the same group, resulting in clobbered HardState). for term := uint64(1); term < magicIters; term++ { if err := mtc.stores[0].HandleRaftMessage(&storage.RaftMessageRequest{ RangeID: trigger.NewDesc.RangeID, ToReplica: replicas[0], FromReplica: replicas[1], Message: raftpb.Message{ Type: raftpb.MsgVote, To: uint64(replicas[0].ReplicaID), From: uint64(replicas[1].ReplicaID), Term: term, }, }); err != nil { t.Error(err) } } }() wg.Wait() } }
func TestSchemaChangeProcess(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. defer csql.TestDisableAsyncSchemaChangeExec()() server, sqlDB, kvDB := setup(t) defer cleanup(server, sqlDB) var id = sqlbase.ID(keys.MaxReservedDescID + 2) var node = roachpb.NodeID(2) db := server.DB() leaseMgr := csql.NewLeaseManager(0, *db, hlc.NewClock(hlc.UnixNano), csql.LeaseManagerTestingKnobs{}) changer := csql.NewSchemaChangerForTesting(id, 0, node, *db, leaseMgr) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, INDEX foo(v)); INSERT INTO t.test VALUES ('a', 'b'), ('c', 'd'); `); err != nil { t.Fatal(err) } // Read table descriptor for version. nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} // Check that MaybeIncrementVersion doesn't increment the version // when the up_version bit is not set. if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } expectedVersion := desc.GetTable().Version desc, err = changer.MaybeIncrementVersion() if err != nil { t.Fatal(err) } newVersion := desc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } isDone, err := changer.IsDone() if err != nil { t.Fatal(err) } if !isDone { t.Fatalf("table expected to not have an outstanding schema change: %v", desc.GetTable()) } // Check that MaybeIncrementVersion increments the version // correctly. expectedVersion++ desc.GetTable().UpVersion = true if err := kvDB.Put(descKey, desc); err != nil { t.Fatal(err) } isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if isDone { t.Fatalf("table expected to have an outstanding schema change: %v", desc.GetTable()) } desc, err = changer.MaybeIncrementVersion() if err != nil { t.Fatal(err) } savedDesc := &sqlbase.Descriptor{} if err := kvDB.GetProto(descKey, savedDesc); err != nil { t.Fatal(err) } newVersion = desc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version in returned desc; e = %d, v = %d", expectedVersion, newVersion) } newVersion = savedDesc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version in saved desc; e = %d, v = %d", expectedVersion, newVersion) } isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if !isDone { t.Fatalf("table expected to not have an outstanding schema change: %v", desc.GetTable()) } // Check that RunStateMachineBeforeBackfill doesn't do anything // if there are no mutations queued. if err := changer.RunStateMachineBeforeBackfill(); err != nil { t.Fatal(err) } if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } newVersion = desc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } // Check that RunStateMachineBeforeBackfill functions properly. if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } table := desc.GetTable() expectedVersion = table.Version // Make a copy of the index for use in a mutation. index := protoutil.Clone(&table.Indexes[0]).(*sqlbase.IndexDescriptor) index.Name = "bar" index.ID = table.NextIndexID table.NextIndexID++ changer = csql.NewSchemaChangerForTesting(id, table.NextMutationID, node, *db, leaseMgr) table.Mutations = append(table.Mutations, sqlbase.DescriptorMutation{ Descriptor_: &sqlbase.DescriptorMutation_Index{Index: index}, Direction: sqlbase.DescriptorMutation_ADD, State: sqlbase.DescriptorMutation_DELETE_ONLY, MutationID: table.NextMutationID, }) table.NextMutationID++ // Run state machine in both directions. for _, direction := range []sqlbase.DescriptorMutation_Direction{sqlbase.DescriptorMutation_ADD, sqlbase.DescriptorMutation_DROP} { table.Mutations[0].Direction = direction expectedVersion++ if err := kvDB.Put(descKey, desc); err != nil { t.Fatal(err) } // The expected end state. expectedState := sqlbase.DescriptorMutation_WRITE_ONLY if direction == sqlbase.DescriptorMutation_DROP { expectedState = sqlbase.DescriptorMutation_DELETE_ONLY } // Run two times to ensure idempotency of operations. for i := 0; i < 2; i++ { if err := changer.RunStateMachineBeforeBackfill(); err != nil { t.Fatal(err) } if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } table = desc.GetTable() newVersion = table.Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } state := table.Mutations[0].State if state != expectedState { t.Fatalf("bad state; e = %d, v = %d", expectedState, state) } } } // RunStateMachineBeforeBackfill() doesn't complete the schema change. isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if isDone { t.Fatalf("table expected to have an outstanding schema change: %v", desc.GetTable()) } }