func TestLeaseManagerDrain(testingT *testing.T) {
defer leaktest.AfterTest(testingT)()
params, _ := createTestServerParams()
leaseRemovalTracker := csql.NewLeaseRemovalTracker()
params.Knobs = base.TestingKnobs{
SQLLeaseManager: &csql.LeaseManagerTestingKnobs{
LeaseStoreTestingKnobs: csql.LeaseStoreTestingKnobs{
LeaseReleasedEvent: leaseRemovalTracker.LeaseRemovedNotification,
},
},
}
t := newLeaseTest(testingT, params)
defer t.cleanup()
const descID = keys.LeaseTableID
{
l1 := t.mustAcquire(1, descID, 0)
l2 := t.mustAcquire(2, descID, 0)
t.mustRelease(1, l1, nil)
t.expectLeases(descID, "/1/1 /1/2")
// Removal tracker to track for node 1's lease removal once the node
// starts draining.
l1RemovalTracker := leaseRemovalTracker.TrackRemoval(l1)
t.nodes[1].SetDraining(true)
t.nodes[2].SetDraining(true)
// Leases cannot be acquired when in draining mode.
if _, err := t.acquire(1, descID, 0); !testutils.IsError(err, "cannot acquire lease when draining") {
t.Fatalf("unexpected error: %v", err)
}
// Node 1's lease has a refcount of 0 and should therefore be removed from
// the store.
if err := l1RemovalTracker.WaitForRemoval(); err != nil {
t.Fatal(err)
}
t.expectLeases(descID, "/1/2")
// Once node 2's lease is released, the lease should be removed from the
// store.
t.mustRelease(2, l2, leaseRemovalTracker)
t.expectLeases(descID, "")
}
{
// Check that leases with a refcount of 0 are correctly kept in the
// store once the drain mode has been exited.
t.nodes[1].SetDraining(false)
l1 := t.mustAcquire(1, descID, 0)
t.mustRelease(1, l1, nil)
t.expectLeases(descID, "/1/1")
}
}
func TestLeaseManagerReacquire(testingT *testing.T) {
defer leaktest.AfterTest(testingT)()
params, _ := createTestServerParams()
removalTracker := csql.NewLeaseRemovalTracker()
params.Knobs = base.TestingKnobs{
SQLLeaseManager: &csql.LeaseManagerTestingKnobs{
LeaseStoreTestingKnobs: csql.LeaseStoreTestingKnobs{
LeaseReleasedEvent: removalTracker.LeaseRemovedNotification,
},
},
}
t := newLeaseTest(testingT, params)
defer t.cleanup()
const descID = keys.LeaseTableID
// Acquire 2 leases from the same node. They should point to the same lease
// structure.
l1 := t.mustAcquire(1, descID, 0)
l2 := t.mustAcquire(1, descID, 0)
if l1 != l2 {
t.Fatalf("expected same lease, but found %p != %p", l1, l2)
}
if l1.Refcount() != 2 {
t.Fatalf("expected refcount of 2, but found %d", l1.Refcount())
}
t.expectLeases(descID, "/1/1")
// Set the minimum lease duration such that the next lease acquisition will
// require the lease to be reacquired.
savedLeaseDuration, savedMinLeaseDuration := csql.LeaseDuration, csql.MinLeaseDuration
defer func() {
csql.LeaseDuration, csql.MinLeaseDuration = savedLeaseDuration, savedMinLeaseDuration
}()
csql.MinLeaseDuration = l1.Expiration().Sub(timeutil.Now())
csql.LeaseDuration = 2 * csql.MinLeaseDuration
// Another lease acquisition from the same node will result in a new lease.
l3 := t.mustAcquire(1, descID, 0)
if l1 == l3 {
t.Fatalf("expected different leases, but found %p", l1)
}
if l3.Refcount() != 1 {
t.Fatalf("expected refcount of 1, but found %d", l3.Refcount())
}
if l3.Expiration().Before(l1.Expiration()) {
t.Fatalf("expected new lease expiration (%s) to be after old lease expiration (%s)",
l3.Expiration(), l1.Expiration())
}
t.expectLeases(descID, "/1/1 /1/1")
t.mustRelease(1, l1, nil)
t.mustRelease(1, l2, removalTracker)
t.mustRelease(1, l3, nil)
}
func TestLeaseManager(testingT *testing.T) {
defer leaktest.AfterTest(testingT)()
removalTracker := csql.NewLeaseRemovalTracker()
params, _ := createTestServerParams()
params.Knobs = base.TestingKnobs{
SQLLeaseManager: &csql.LeaseManagerTestingKnobs{
LeaseStoreTestingKnobs: csql.LeaseStoreTestingKnobs{
LeaseReleasedEvent: removalTracker.LeaseRemovedNotification,
},
},
}
t := newLeaseTest(testingT, params)
defer t.cleanup()
const descID = keys.LeaseTableID
// We can't acquire a lease on a non-existent table.
expected := "descriptor not found"
if _, err := t.acquire(1, 10000, 0); !testutils.IsError(err, expected) {
t.Fatalf("expected %s, but found %v", expected, err)
}
l1 := t.mustAcquire(1, descID, 0)
t.expectLeases(descID, "/1/1")
// Node 2 never acquired a lease on descID, so we should expect an error.
if err := t.release(2, l1); err == nil {
t.Fatalf("expected error, but found none")
}
t.mustRelease(1, l1, nil)
t.expectLeases(descID, "/1/1")
// It is an error to acquire a lease for a specific version that doesn't
// exist yet.
expected = "version 2 of table .* does not exist"
if _, err := t.acquire(1, descID, 2); !testutils.IsError(err, expected) {
t.Fatalf("expected %s, but found %v", expected, err)
}
// Publish a new version and explicitly acquire it.
l2 := t.mustAcquire(1, descID, 0)
t.mustPublish(1, descID)
l3 := t.mustAcquire(1, descID, 2)
t.expectLeases(descID, "/1/1 /2/1")
// When the last local reference on the new version is released we don't
// release the node lease.
t.mustRelease(1, l3, nil)
t.expectLeases(descID, "/1/1 /2/1")
// We can still acquire a local reference on the old version since it hasn't
// expired.
l4 := t.mustAcquire(1, descID, 1)
t.mustRelease(1, l4, nil)
t.expectLeases(descID, "/1/1 /2/1")
// When the last local reference on the old version is released the node
// lease is also released.
t.mustRelease(1, l2, removalTracker)
t.expectLeases(descID, "/2/1")
// It is an error to acquire a lease for an old version once a new version
// exists and there are no local references for the old version.
expected = `table \d+ unable to acquire lease on old version: 1 < 2`
if _, err := t.acquire(1, descID, 1); !testutils.IsError(err, expected) {
t.Fatalf("expected %s, but found %v", expected, err)
}
// Acquire 2 node leases on version 2.
l5 := t.mustAcquire(1, descID, 2)
l6 := t.mustAcquire(2, descID, 2)
// Publish version 3. This will succeed immediately.
t.mustPublish(3, descID)
// Start a goroutine to publish version 4 which will block until the version
// 2 leases are released.
var wg sync.WaitGroup
wg.Add(1)
go func() {
t.mustPublish(3, descID)
wg.Done()
}()
// Force both nodes ahead to version 3.
l7 := t.mustAcquire(1, descID, 3)
l8 := t.mustAcquire(2, descID, 3)
t.expectLeases(descID, "/2/1 /2/2 /3/1 /3/2")
t.mustRelease(1, l5, removalTracker)
t.expectLeases(descID, "/2/2 /3/1 /3/2")
t.mustRelease(2, l6, removalTracker)
t.expectLeases(descID, "/3/1 /3/2")
// Wait for version 4 to be published.
wg.Wait()
l9 := t.mustAcquire(1, descID, 4)
t.mustRelease(1, l7, removalTracker)
t.mustRelease(2, l8, nil)
t.expectLeases(descID, "/3/2 /4/1")
t.mustRelease(1, l9, nil)
t.expectLeases(descID, "/3/2 /4/1")
}