func TestLeaderLeaseSwapWhileWaiting(t *testing.T) {
testutil.RequireEtcd(t)
client := testutil.NewEtcdClient()
key := "/random/key"
if _, err := client.Create(key, "holder", 10); err != nil {
t.Fatal(err)
}
go func() {
time.Sleep(time.Second)
if _, err := client.Set(key, "other", 10); err != nil {
t.Fatal(err)
}
glog.Infof("Changed key ownership")
}()
lease := leaderlease.NewEtcd(client, key, "other", 10)
ch := make(chan struct{})
go lease.AcquireAndHold(ch)
<-ch
glog.Infof("Lease acquired")
lease.Release()
<-ch
glog.Infof("Lease gone")
}
func TestLeaderLeaseSwapWhileWaiting(t *testing.T) {
testutil.RequireEtcd(t)
defer testutil.DumpEtcdOnFailure(t)
c, err := testutil.MakeNewEtcdClient()
if err != nil {
t.Fatal(err)
}
client := etcdclient.NewKeysAPI(c)
key := "/random/key"
if _, err := client.Set(context.Background(), key, "holder", &etcdclient.SetOptions{TTL: 10 * time.Second, PrevExist: etcdclient.PrevNoExist}); err != nil {
t.Fatal(err)
}
go func() {
time.Sleep(time.Second)
if _, err := client.Set(context.Background(), key, "other", &etcdclient.SetOptions{TTL: 10 * time.Second}); err != nil {
t.Fatal(err)
}
glog.Infof("Changed key ownership")
}()
lease := leaderlease.NewEtcd(c, key, "other", 10)
ch := make(chan error, 1)
go lease.AcquireAndHold(ch)
<-ch
glog.Infof("Lease acquired")
lease.Release()
if err, ok := <-ch; err == nil || !ok || !strings.Contains(err.Error(), "the lease has been lost") {
t.Errorf("Expected error and open channel when lease was swapped: %v %t", err, ok)
}
<-ch
glog.Infof("Lease gone")
}
func TestLeaderLeaseAcquire(t *testing.T) {
testutil.RequireEtcd(t)
client := testutil.NewEtcdClient()
key := "/random/key"
held := make(chan struct{})
go func() {
<-held
if _, err := client.Delete(key, false); err != nil {
t.Fatal(err)
}
glog.Infof("Deleted key")
}()
lease := leaderlease.NewEtcd(client, key, "holder", 10)
ch := make(chan struct{})
go lease.AcquireAndHold(ch)
<-ch
glog.Infof("Lease acquired")
close(held)
<-ch
glog.Infof("Lease lost")
select {
case _, ok := <-held:
if ok {
t.Error("did not acquire the lease")
}
default:
t.Error("lease is still open")
}
}
func TestLeaderLeaseSwapWhileWaiting(t *testing.T) {
testutil.RequireEtcd(t)
defer testutil.DumpEtcdOnFailure(t)
client := testutil.NewEtcdClient()
key := "/random/key"
if _, err := client.Create(key, "holder", 10); err != nil {
t.Fatal(err)
}
go func() {
time.Sleep(time.Second)
if _, err := client.Set(key, "other", 10); err != nil {
t.Fatal(err)
}
glog.Infof("Changed key ownership")
}()
lease := leaderlease.NewEtcd(client, key, "other", 10)
ch := make(chan error, 1)
go lease.AcquireAndHold(ch)
<-ch
glog.Infof("Lease acquired")
lease.Release()
if err, ok := <-ch; err == nil || !ok || !strings.Contains(err.Error(), "the lease has been lost") {
t.Errorf("Expected error and open channel when lease was swapped: %v %t", err, ok)
}
<-ch
glog.Infof("Lease gone")
}
func TestLeaderLeaseWait(t *testing.T) {
testutil.RequireEtcd(t)
defer testutil.DumpEtcdOnFailure(t)
c, err := testutil.MakeNewEtcdClient()
if err != nil {
t.Fatal(err)
}
client := etcdclient.NewKeysAPI(c)
key := "/random/key"
if _, err := client.Set(context.Background(), key, "other", &etcdclient.SetOptions{TTL: time.Second, PrevExist: etcdclient.PrevNoExist}); err != nil {
t.Fatal(err)
}
held := make(chan struct{})
go func() {
<-held
if _, err := client.Delete(context.Background(), key, nil); err != nil {
t.Fatal(err)
}
glog.Infof("Deleted key")
}()
lease := leaderlease.NewEtcd(c, key, "holder", 10)
ch := make(chan error, 1)
go lease.AcquireAndHold(ch)
<-ch
glog.Infof("Lease acquired")
close(held)
if err, ok := <-ch; err == nil || !ok || !strings.Contains(err.Error(), "the lease has been lost") {
t.Errorf("Expected error and open channel when lease was swapped: %v %t", err, ok)
}
<-ch
glog.Infof("Lease lost")
select {
case _, ok := <-held:
if ok {
t.Error("did not acquire the lease")
}
default:
t.Error("lease is still open")
}
}
func TestLeaderLeaseReacquire(t *testing.T) {
testutil.RequireEtcd(t)
defer testutil.DumpEtcdOnFailure(t)
c, err := testutil.MakeNewEtcdClient()
if err != nil {
t.Fatal(err)
}
client := etcdclient.NewKeysAPI(c)
key := "/random/key"
if _, err := client.Set(context.Background(), key, "holder", &etcdclient.SetOptions{TTL: time.Second, PrevExist: etcdclient.PrevNoExist}); err != nil {
t.Fatal(err)
}
held := make(chan struct{})
go func() {
<-held
if _, err := client.Delete(context.Background(), key, nil); err != nil {
t.Fatal(err)
}
glog.Infof("Deleted key")
}()
lease := leaderlease.NewEtcd(c, key, "holder", 1)
ch := make(chan error, 1)
go lease.AcquireAndHold(ch)
<-ch
glog.Infof("Lease acquired")
time.Sleep(2 * time.Second)
close(held)
<-ch
glog.Infof("Lease lost")
select {
case _, ok := <-held:
if ok {
t.Error("did not acquire the lease")
}
default:
t.Error("lease is still open")
}
}
func newControllerPlug(options configapi.MasterConfig, client *etcdclient.Client) (plug.Plug, func()) {
switch {
case options.ControllerLeaseTTL > 0:
// TODO: replace with future API for leasing from Kube
id := fmt.Sprintf("master-%s", kutilrand.String(8))
leaser := leaderlease.NewEtcd(
client,
path.Join(options.EtcdStorageConfig.OpenShiftStoragePrefix, "leases/controllers"),
id,
uint64(options.ControllerLeaseTTL),
)
leased := plug.NewLeased(leaser)
return leased, func() {
glog.V(2).Infof("Attempting to acquire controller lease as %s, renewing every %d seconds", id, options.ControllerLeaseTTL)
go leased.Run()
}
default:
return plug.New(!options.PauseControllers), func() {}
}
}
func TestLeaderLeaseWait(t *testing.T) {
testutil.RequireEtcd(t)
defer testutil.DumpEtcdOnFailure(t)
client := testutil.NewEtcdClient()
key := "/random/key"
if _, err := client.Create(key, "other", 1); err != nil {
t.Fatal(err)
}
held := make(chan struct{})
go func() {
<-held
if _, err := client.Delete(key, false); err != nil {
t.Fatal(err)
}
glog.Infof("Deleted key")
}()
lease := leaderlease.NewEtcd(client, key, "holder", 10)
ch := make(chan error, 1)
go lease.AcquireAndHold(ch)
<-ch
glog.Infof("Lease acquired")
close(held)
if err, ok := <-ch; err == nil || !ok || !strings.Contains(err.Error(), "the lease has been lost") {
t.Errorf("Expected error and open channel when lease was swapped: %v %t", err, ok)
}
<-ch
glog.Infof("Lease lost")
select {
case _, ok := <-held:
if ok {
t.Error("did not acquire the lease")
}
default:
t.Error("lease is still open")
}
}
func TestLeaderLeaseReacquire(t *testing.T) {
util.DeleteAllEtcdKeys()
client := util.NewEtcdClient()
key := "/random/key"
if _, err := client.Create(key, "holder", 1); err != nil {
t.Fatal(err)
}
held := make(chan struct{})
go func() {
<-held
if _, err := client.Delete(key, false); err != nil {
t.Fatal(err)
}
glog.Infof("Deleted key")
}()
lease := leaderlease.NewEtcd(client, key, "holder", 1)
ch := make(chan struct{})
go lease.AcquireAndHold(ch)
<-ch
glog.Infof("Lease acquired")
time.Sleep(2 * time.Second)
close(held)
<-ch
glog.Infof("Lease lost")
select {
case _, ok := <-held:
if ok {
t.Error("did not acquire the lease")
}
default:
t.Error("lease is still open")
}
}