func (p *planner) releaseLeases(db client.DB) {
if p.leases != nil {
for _, lease := range p.leases {
if err := p.leaseMgr.Release(lease); err != nil {
log.Warning(err)
}
}
p.leases = nil
}
// TODO(pmattis): This is a hack. Remove when schema change operations work
// properly.
if p.modifiedSchemas != nil {
for _, d := range p.modifiedSchemas {
var lease *LeaseState
err := db.Txn(func(txn *client.Txn) error {
var err error
lease, err = p.leaseMgr.Acquire(txn, d.id, d.version)
return err
})
if err != nil {
log.Warning(err)
continue
}
if err := p.leaseMgr.Release(lease); err != nil {
log.Warning(err)
}
}
p.modifiedSchemas = nil
}
}
// purgeOldLeases refreshes the leases on a table. Unused leases older than
// minVersion will be released.
// If deleted is set, minVersion is ignored; no lease is acquired and all
// existing unused leases are released. The table is further marked for
// deletion, which will cause existing in-use leases to be eagerly released once
// they're not in use any more.
// If t has no active leases, nothing is done.
func (t *tableState) purgeOldLeases(
db *client.DB, deleted bool, minVersion sqlbase.DescriptorVersion, store LeaseStore,
) error {
t.mu.Lock()
empty := len(t.active.data) == 0
t.mu.Unlock()
if empty {
// We don't currently have a lease on this table, so no need to refresh
// anything.
return nil
}
// Acquire and release a lease on the table at a version >= minVersion.
var lease *LeaseState
err := db.Txn(func(txn *client.Txn) error {
var err error
if !deleted {
lease, err = t.acquire(txn, minVersion, store)
if err == errTableDeleted {
deleted = true
}
}
if err == nil || deleted {
t.mu.Lock()
defer t.mu.Unlock()
var toRelease []*LeaseState
if deleted {
t.deleted = true
// If the table has been deleted, all leases are stale.
toRelease = append([]*LeaseState(nil), t.active.data...)
} else {
// Otherwise, all but the lease we just took are stale.
toRelease = append([]*LeaseState(nil), t.active.data[:len(t.active.data)-1]...)
}
if err := t.releaseLeasesIfNotActive(toRelease, store); err != nil {
return err
}
return nil
}
return err
})
if err != nil {
return err
}
if lease == nil {
return nil
}
return t.release(lease, store)
}
func (hv *historyVerifier) runCmds(cmds []*cmd, historyIdx int, db *client.DB, t *testing.T) (string, map[string]int64, error) {
var strs []string
env := map[string]int64{}
err := db.Txn(func(txn *client.Txn) error {
for _, c := range cmds {
c.historyIdx = historyIdx
c.env = env
c.init(nil)
fmtStr, err := c.execute(txn, t)
if err != nil {
return err
}
strs = append(strs, fmt.Sprintf(fmtStr, 0, 0))
}
return nil
})
return strings.Join(strs, " "), env, err
}
// startTestWriter creates a writer which initiates a sequence of
// transactions, each which writes up to 10 times to random keys with
// random values. If not nil, txnChannel is written to non-blockingly
// every time a new transaction starts.
func startTestWriter(db *client.DB, i int64, valBytes int32, wg *sync.WaitGroup, retries *int32,
txnChannel chan struct{}, done <-chan struct{}, t *testing.T) {
src := rand.New(rand.NewSource(i))
defer func() {
if wg != nil {
wg.Done()
}
}()
for j := 0; ; j++ {
select {
case <-done:
return
default:
first := true
err := db.Txn(func(txn *client.Txn) error {
if first && txnChannel != nil {
select {
case txnChannel <- struct{}{}:
default:
}
} else if !first && retries != nil {
atomic.AddInt32(retries, 1)
}
first = false
for j := 0; j <= int(src.Int31n(10)); j++ {
key := randutil.RandBytes(src, 10)
val := randutil.RandBytes(src, int(src.Int31n(valBytes)))
if err := txn.Put(key, val); err != nil {
log.Infof("experienced an error in routine %d: %s", i, err)
return err
}
}
return nil
})
if err != nil {
t.Error(err)
} else {
time.Sleep(1 * time.Millisecond)
}
}
}
}
func (hv *historyVerifier) runTxn(txnIdx int, priority int32,
isolation roachpb.IsolationType, cmds []*cmd, db *client.DB, t *testing.T) error {
var retry int
txnName := fmt.Sprintf("txn%d", txnIdx)
err := db.Txn(func(txn *client.Txn) error {
txn.SetDebugName(txnName, 0)
if isolation == roachpb.SNAPSHOT {
if err := txn.SetIsolation(roachpb.SNAPSHOT); err != nil {
return err
}
}
txn.InternalSetPriority(priority)
env := map[string]int64{}
// TODO(spencer): restarts must create additional histories. They
// look like: given the current partial history and a restart on
// txn txnIdx, re-enumerate a set of all histories containing the
// remaining commands from extant txns and all commands from this
// restarted txn.
// If this is attempt > 1, reset cmds so no waits.
if retry++; retry == 2 {
for _, c := range cmds {
c.done()
}
}
if log.V(2) {
log.Infof("%s, retry=%d", txnName, retry)
}
for i := range cmds {
cmds[i].env = env
if err := hv.runCmd(txn, txnIdx, retry, i, cmds, t); err != nil {
if log.V(1) {
log.Infof("%s encountered error: %s", cmds[i], err)
}
return err
}
}
return nil
})
hv.wg.Done()
return err
}
// refreshLease tries to refresh the node's table lease.
func (m *LeaseManager) refreshLease(db *client.DB, id ID, minVersion DescriptorVersion) error {
// Only attempt to update a lease for a table that is already leased.
if t := m.findTableState(id, false); t == nil {
return nil
}
// Acquire and release a lease on the table at a version >= minVersion.
var lease *LeaseState
if pErr := db.Txn(func(txn *client.Txn) *roachpb.Error {
var pErr *roachpb.Error
// Acquire() can only acquire a lease at a version if it has
// already been acquired at that version, or that version
// is the latest version. If the latest version is > minVersion
// then the node acquires a lease at the latest version but
// Acquire() itself returns an error. This is okay, because
// we want to update the node lease.
lease, pErr = m.Acquire(txn, id, minVersion)
return pErr
}); pErr != nil {
return pErr.GoError()
}
return m.Release(lease)
}
func (hv *historyVerifier) runHistory(historyIdx int, priorities []int32,
isolations []roachpb.IsolationType, cmds []*cmd, db *client.DB, t *testing.T) error {
plannedStr := historyString(cmds)
if log.V(1) {
log.Infof("attempting iso=%v pri=%v history=%s", isolations, priorities, plannedStr)
}
hv.actual = []string{}
hv.wg.Add(len(priorities))
txnMap := map[int][]*cmd{}
var prev *cmd
for _, c := range cmds {
c.historyIdx = historyIdx
txnMap[c.txnIdx] = append(txnMap[c.txnIdx], c)
c.init(prev)
prev = c
}
for i, txnCmds := range txnMap {
go func(i int, txnCmds []*cmd) {
if err := hv.runTxn(i, priorities[i-1], isolations[i-1], txnCmds, db, t); err != nil {
t.Errorf("(%s): unexpected failure running %s: %v", cmds, cmds[i], err)
}
}(i, txnCmds)
}
hv.wg.Wait()
// Construct string for actual history.
actualStr := strings.Join(hv.actual, " ")
// Verify history.
var verifyStrs []string
verifyEnv := map[string]int64{}
for _, c := range hv.verifyCmds {
c.historyIdx = historyIdx
c.env = verifyEnv
c.init(nil)
pErr := db.Txn(func(txn *client.Txn) *roachpb.Error {
fmtStr, pErr := c.execute(txn, t)
if pErr != nil {
return pErr
}
cmdStr := fmt.Sprintf(fmtStr, 0, 0)
verifyStrs = append(verifyStrs, cmdStr)
return nil
})
if pErr != nil {
t.Errorf("failed on execution of verification cmd %s: %s", c, pErr)
return pErr.GoError()
}
}
err := hv.verify.checkFn(verifyEnv)
if err == nil {
if log.V(1) {
log.Infof("PASSED: iso=%v, pri=%v, history=%q", isolations, priorities, actualStr)
}
}
if hv.expSuccess && err != nil {
verifyStr := strings.Join(verifyStrs, " ")
t.Errorf("%d: iso=%v, pri=%v, history=%q: actual=%q, verify=%q: %s",
historyIdx, isolations, priorities, plannedStr, actualStr, verifyStr, err)
}
return err
}
// concurrentIncrements starts two Goroutines in parallel, both of which
// read the integers stored at the other's key and add it onto their own.
// It is checked that the outcome is serializable, i.e. exactly one of the
// two Goroutines (the later write) sees the previous write by the other.
func concurrentIncrements(db *client.DB, t *testing.T) {
// wgStart waits for all transactions to line up, wgEnd has the main
// function wait for them to finish.
var wgStart, wgEnd sync.WaitGroup
wgStart.Add(2 + 1)
wgEnd.Add(2)
for i := 0; i < 2; i++ {
go func(i int) {
// Read the other key, write key i.
readKey := []byte(fmt.Sprintf(testUser+"/value-%d", (i+1)%2))
writeKey := []byte(fmt.Sprintf(testUser+"/value-%d", i))
defer wgEnd.Done()
wgStart.Done()
// Wait until the other goroutines are running.
wgStart.Wait()
if pErr := db.Txn(func(txn *client.Txn) *roachpb.Error {
txn.SetDebugName(fmt.Sprintf("test-%d", i), 0)
// Retrieve the other key.
gr, pErr := txn.Get(readKey)
if pErr != nil {
return pErr
}
otherValue := int64(0)
if gr.Value != nil {
otherValue = gr.ValueInt()
}
_, pErr = txn.Inc(writeKey, 1+otherValue)
return pErr
}); pErr != nil {
t.Error(pErr)
}
}(i)
}
// Kick the goroutines loose.
wgStart.Done()
// Wait for the goroutines to finish.
wgEnd.Wait()
// Verify that both keys contain something and, more importantly, that
// one key actually contains the value of the first writer and not only
// its own.
total := int64(0)
results := []int64(nil)
for i := 0; i < 2; i++ {
readKey := []byte(fmt.Sprintf(testUser+"/value-%d", i))
gr, pErr := db.Get(readKey)
if pErr != nil {
t.Fatal(pErr)
}
if gr.Value == nil {
t.Fatalf("unexpected empty key: %s=%v", readKey, gr.Value)
}
total += gr.ValueInt()
results = append(results, gr.ValueInt())
}
// First writer should have 1, second one 2
if total != 3 {
t.Fatalf("got unserializable values %v", results)
}
}
// concurrentIncrements starts two Goroutines in parallel, both of which
// read the integers stored at the other's key and add it onto their own.
// It is checked that the outcome is serializable, i.e. exactly one of the
// two Goroutines (the later write) sees the previous write by the other.
// The isMultiphase option runs the transaction in multiple phases recreating
// the transaction from the transaction protobuf returned from the server.
func concurrentIncrements(db *client.DB, t *testing.T, isMultiphase bool) {
// wgStart waits for all transactions to line up, wgEnd has the main
// function wait for them to finish.
var wgStart, wgEnd sync.WaitGroup
wgStart.Add(2 + 1)
wgEnd.Add(2)
for i := 0; i < 2; i++ {
go func(i int) {
// Read the other key, write key i.
readKey := []byte(fmt.Sprintf(testUser+"/value-%d", (i+1)%2))
writeKey := []byte(fmt.Sprintf(testUser+"/value-%d", i))
defer wgEnd.Done()
wgStart.Done()
// Wait until the other goroutines are running.
wgStart.Wait()
if isMultiphase {
applyInc := func(txn *client.Txn) (error, proto.Transaction) {
txn.SetDebugName(fmt.Sprintf("test-%d", i))
b := client.Batch{}
// Retrieve the other key.
b.Get(readKey)
if err := txn.Run(&b); err != nil {
return err, txn.GetState()
}
otherValue := int64(0)
gr := b.Results[0].Rows[0]
if gr.Value != nil {
otherValue = gr.ValueInt()
}
// New txn.
txn = db.ReconstructTxn(txn.GetState())
// Write our key.
b = client.Batch{}
b.Inc(writeKey, 1+otherValue)
if err := txn.Run(&b); err != nil {
return err, txn.GetState()
}
// New txn.
txn = db.ReconstructTxn(txn.GetState())
err := txn.Commit(&client.Batch{})
return err, txn.GetState()
}
for r := retry.Start(client.DefaultTxnRetryOptions); r.Next(); {
txn := db.ReconstructTxn(proto.Transaction{})
if err, txnProto := applyInc(txn); err != nil {
// New txn.
txn = db.ReconstructTxn(txnProto)
if err := txn.Rollback(); err != nil {
t.Error(err)
} else {
// retry
continue
}
}
// exit retry
break
}
} else {
if err := db.Txn(func(txn *client.Txn) error {
txn.SetDebugName(fmt.Sprintf("test-%d", i))
// Retrieve the other key.
gr, err := txn.Get(readKey)
if err != nil {
return err
}
otherValue := int64(0)
if gr.Value != nil {
otherValue = gr.ValueInt()
}
_, err = txn.Inc(writeKey, 1+otherValue)
return err
}); err != nil {
t.Error(err)
}
}
}(i)
}
// Kick the goroutines loose.
wgStart.Done()
// Wait for the goroutines to finish.
wgEnd.Wait()
// Verify that both keys contain something and, more importantly, that
// one key actually contains the value of the first writer and not only
// its own.
total := int64(0)
results := []int64(nil)
for i := 0; i < 2; i++ {
readKey := []byte(fmt.Sprintf(testUser+"/value-%d", i))
gr, err := db.Get(readKey)
if err != nil {
log.Fatal(err)
}
if gr.Value == nil {
t.Fatalf("unexpected empty key: %s=%v", readKey, gr.Value)
}
total += gr.ValueInt()
results = append(results, gr.ValueInt())
}
// First writer should have 1, second one 2
if total != 3 {
t.Fatalf("got unserializable values %v", results)
}
}