func TestDoubleBarrierFailover(t *testing.T) {
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
defer clus.Terminate(t)
waiters := 10
donec := make(chan struct{})
s0, err := concurrency.NewSession(clus.clients[0])
if err != nil {
t.Error(err)
}
defer s0.Orphan()
s1, err := concurrency.NewSession(clus.clients[0])
if err != nil {
t.Error(err)
}
defer s1.Orphan()
// sacrificial barrier holder; lease will be revoked
go func() {
b := recipe.NewDoubleBarrier(s0, "test-barrier", waiters)
if berr := b.Enter(); berr != nil {
t.Fatalf("could not enter on barrier (%v)", berr)
}
donec <- struct{}{}
}()
for i := 0; i < waiters-1; i++ {
go func() {
b := recipe.NewDoubleBarrier(s1, "test-barrier", waiters)
if berr := b.Enter(); berr != nil {
t.Fatalf("could not enter on barrier (%v)", berr)
}
donec <- struct{}{}
b.Leave()
donec <- struct{}{}
}()
}
// wait for barrier enter to unblock
for i := 0; i < waiters; i++ {
select {
case <-donec:
case <-time.After(10 * time.Second):
t.Fatalf("timed out waiting for enter, %d", i)
}
}
if err = s0.Close(); err != nil {
t.Fatal(err)
}
// join on rest of waiters
for i := 0; i < waiters-1; i++ {
select {
case <-donec:
case <-time.After(10 * time.Second):
t.Fatalf("timed out waiting for leave, %d", i)
}
}
}
func doSTM(ctx context.Context, client *v3.Client, requests <-chan stmApply) {
defer wg.Done()
var m *v3sync.Mutex
if stmMutex {
s, err := v3sync.NewSession(client)
if err != nil {
panic(err)
}
m = v3sync.NewMutex(s, "stmlock")
}
for applyf := range requests {
st := time.Now()
if m != nil {
m.Lock(context.TODO())
}
_, err := mkSTM(context.TODO(), client, applyf)
if m != nil {
m.Unlock(context.TODO())
}
var errStr string
if err != nil {
errStr = err.Error()
}
results <- result{errStr: errStr, duration: time.Since(st), happened: time.Now()}
bar.Increment()
}
}
func runRacer(getClient getClientFunc, round int) {
rcs := make([]roundClient, 15)
ctx := context.Background()
cnt := 0
for i := range rcs {
rcs[i].c = getClient()
var (
s *concurrency.Session
err error
)
for {
s, err = concurrency.NewSession(rcs[i].c)
if err == nil {
break
}
}
m := concurrency.NewMutex(s, "racers")
rcs[i].acquire = func() error { return m.Lock(ctx) }
rcs[i].validate = func() error {
if cnt++; cnt != 1 {
return fmt.Errorf("bad lock; count: %d", cnt)
}
return nil
}
rcs[i].release = func() error {
if err := m.Unlock(ctx); err != nil {
return err
}
cnt = 0
return nil
}
}
doRounds(rcs, round)
}
func observe(c *clientv3.Client, election string) error {
s, err := concurrency.NewSession(c)
if err != nil {
return err
}
e := concurrency.NewElection(s, election)
ctx, cancel := context.WithCancel(context.TODO())
donec := make(chan struct{})
sigc := make(chan os.Signal, 1)
signal.Notify(sigc, os.Interrupt, os.Kill)
go func() {
<-sigc
cancel()
}()
go func() {
for resp := range e.Observe(ctx) {
display.Get(resp)
}
close(donec)
}()
<-donec
select {
case <-ctx.Done():
default:
return errors.New("elect: observer lost")
}
return nil
}
func doSTM(client *v3.Client, requests <-chan stmApply, results chan<- report.Result) {
defer wg.Done()
var m *v3sync.Mutex
if stmMutex {
s, err := v3sync.NewSession(client)
if err != nil {
panic(err)
}
m = v3sync.NewMutex(s, "stmlock")
}
for applyf := range requests {
st := time.Now()
if m != nil {
m.Lock(context.TODO())
}
_, err := mkSTM(context.TODO(), client, applyf)
if m != nil {
m.Unlock(context.TODO())
}
results <- report.Result{Err: err, Start: st, End: time.Now()}
bar.Increment()
}
}
func testMutex(t *testing.T, waiters int, chooseClient func() *clientv3.Client) {
// stream lock acquisitions
lockedC := make(chan *concurrency.Mutex)
for i := 0; i < waiters; i++ {
go func() {
session, err := concurrency.NewSession(chooseClient())
if err != nil {
t.Error(err)
}
m := concurrency.NewMutex(session, "test-mutex")
if err := m.Lock(context.TODO()); err != nil {
t.Fatalf("could not wait on lock (%v)", err)
}
lockedC <- m
}()
}
// unlock locked mutexes
timerC := time.After(time.Duration(waiters) * time.Second)
for i := 0; i < waiters; i++ {
select {
case <-timerC:
t.Fatalf("timed out waiting for lock %d", i)
case m := <-lockedC:
// lock acquired with m
select {
case <-lockedC:
t.Fatalf("lock %d followers did not wait", i)
default:
}
if err := m.Unlock(context.TODO()); err != nil {
t.Fatalf("could not release lock (%v)", err)
}
}
}
}
// TestElectionSessionRelock ensures that campaigning twice on the same election
// with the same lock will Proclaim instead of deadlocking.
func TestElectionSessionRecampaign(t *testing.T) {
clus := NewClusterV3(t, &ClusterConfig{Size: 1})
defer clus.Terminate(t)
cli := clus.RandClient()
session, err := concurrency.NewSession(cli)
if err != nil {
t.Error(err)
}
defer session.Orphan()
e := concurrency.NewElection(session, "test-elect")
if err := e.Campaign(context.TODO(), "abc"); err != nil {
t.Fatal(err)
}
e2 := concurrency.NewElection(session, "test-elect")
if err := e2.Campaign(context.TODO(), "def"); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
if resp := <-e.Observe(ctx); len(resp.Kvs) == 0 || string(resp.Kvs[0].Value) != "def" {
t.Fatalf("expected value=%q, got response %v", "def", resp)
}
}
func lockUntilSignal(c *clientv3.Client, lockname string) error {
s, err := concurrency.NewSession(c)
if err != nil {
return err
}
m := concurrency.NewMutex(s, lockname)
ctx, cancel := context.WithCancel(context.TODO())
// unlock in case of ordinary shutdown
donec := make(chan struct{})
sigc := make(chan os.Signal, 1)
signal.Notify(sigc, os.Interrupt, os.Kill)
go func() {
<-sigc
cancel()
close(donec)
}()
s, serr := concurrency.NewSession(c)
if serr != nil {
return serr
}
if err := m.Lock(ctx); err != nil {
return err
}
k, kerr := c.Get(ctx, m.Key())
if kerr != nil {
return kerr
}
if len(k.Kvs) == 0 {
return errors.New("lock lost on init")
}
display.Get(*k)
select {
case <-donec:
return m.Unlock(context.TODO())
case <-s.Done():
}
return errors.New("session expired")
}
// TestElectionFailover tests that an election will
func TestElectionFailover(t *testing.T) {
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
defer clus.Terminate(t)
defer dropSessionLease(clus)
cctx, cancel := context.WithCancel(context.TODO())
defer cancel()
// first leader (elected)
e := concurrency.NewElection(clus.clients[0], "test-election")
if err := e.Campaign(context.TODO(), "foo"); err != nil {
t.Fatalf("failed volunteer (%v)", err)
}
// check first leader
resp, ok := <-e.Observe(cctx)
if !ok {
t.Fatalf("could not wait for first election; channel closed")
}
s := string(resp.Kvs[0].Value)
if s != "foo" {
t.Fatalf("wrong election result. got %s, wanted foo", s)
}
// next leader
electedc := make(chan struct{})
go func() {
ee := concurrency.NewElection(clus.clients[1], "test-election")
if eer := ee.Campaign(context.TODO(), "bar"); eer != nil {
t.Fatal(eer)
}
electedc <- struct{}{}
}()
// invoke leader failover
session, serr := concurrency.NewSession(clus.clients[0])
if serr != nil {
t.Fatal(serr)
}
if err := session.Close(); err != nil {
t.Fatal(err)
}
// check new leader
e = concurrency.NewElection(clus.clients[2], "test-election")
resp, ok = <-e.Observe(cctx)
if !ok {
t.Fatalf("could not wait for second election; channel closed")
}
s = string(resp.Kvs[0].Value)
if s != "bar" {
t.Fatalf("wrong election result. got %s, wanted bar", s)
}
// leader must ack election (otherwise, Campaign may see closed conn)
<-electedc
}
func NewEtcdClient(config *client.Config, cfgPath string) (KVClient, error) {
var (
c *client.Client
err error
)
if cfgPath != "" {
c, err = client.NewFromConfigFile(cfgPath)
} else if config != nil {
c, err = client.New(*config)
} else {
err = fmt.Errorf("empty configuration provided")
}
if err != nil {
return nil, err
}
log.Info("Waiting for etcd client to be ready...")
s, err := concurrency.NewSession(c)
if err != nil {
return nil, fmt.Errorf("Unable to contact etcd: %s", err)
}
log.Info("Etcd client ready")
ec := &EtcdClient{
cli: c,
session: s,
lockPaths: map[string]*sync.Mutex{},
}
go func() {
for {
<-ec.session.Done()
newSession, err := concurrency.NewSession(c)
if err != nil {
log.Errorf("Error while renewing etcd session %s", err)
time.Sleep(10 * time.Second)
} else {
ec.sessionMU.Lock()
ec.session = newSession
ec.sessionMU.Unlock()
log.Debugf("Renewing etcd session")
}
}
}()
return ec, nil
}
// NewEphemeralKV creates a new key/value pair associated with a session lease
func NewEphemeralKV(client *v3.Client, key, val string) (*EphemeralKV, error) {
s, err := concurrency.NewSession(client)
if err != nil {
return nil, err
}
k, err := NewKV(client, key, val, s.Lease())
if err != nil {
return nil, err
}
return &EphemeralKV{*k}, nil
}
func campaign(c *clientv3.Client, election string, prop string) error {
s, err := concurrency.NewSession(c)
if err != nil {
return err
}
e := concurrency.NewElection(s, election)
ctx, cancel := context.WithCancel(context.TODO())
donec := make(chan struct{})
sigc := make(chan os.Signal, 1)
signal.Notify(sigc, os.Interrupt, os.Kill)
go func() {
<-sigc
cancel()
close(donec)
}()
s, serr := concurrency.NewSession(c)
if serr != nil {
return serr
}
if err = e.Campaign(ctx, prop); err != nil {
return err
}
// print key since elected
resp, err := c.Get(ctx, e.Key())
if err != nil {
return err
}
display.Get(*resp)
select {
case <-donec:
case <-s.Done():
return errors.New("elect: session expired")
}
return e.Resign(context.TODO())
}
// TestElectionFailover tests that an election will
func TestElectionFailover(t *testing.T) {
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
defer clus.Terminate(t)
defer dropSessionLease(clus)
// first leader (elected)
e := recipe.NewElection(clus.clients[0], "test-election")
if err := e.Volunteer("foo"); err != nil {
t.Fatalf("failed volunteer (%v)", err)
}
// check first leader
s, err := e.Wait()
if err != nil {
t.Fatalf("could not wait for first election (%v)", err)
}
if s != "foo" {
t.Fatalf("wrong election result. got %s, wanted foo", s)
}
// next leader
electedc := make(chan struct{})
go func() {
ee := recipe.NewElection(clus.clients[1], "test-election")
if eer := ee.Volunteer("bar"); eer != nil {
t.Fatal(eer)
}
electedc <- struct{}{}
}()
// invoke leader failover
session, serr := concurrency.NewSession(clus.clients[0])
if serr != nil {
t.Fatal(serr)
}
err = session.Close()
if err != nil {
t.Fatal(err)
}
// check new leader
e = recipe.NewElection(clus.clients[2], "test-election")
s, err = e.Wait()
if err != nil {
t.Fatalf("could not wait for second election (%v)", err)
}
if s != "bar" {
t.Fatalf("wrong election result. got %s, wanted bar", s)
}
// leader must ack election (otherwise, Volunteer may see closed conn)
<-electedc
}
func testRWMutex(t *testing.T, waiters int, chooseClient func() *clientv3.Client) {
// stream rwlock acquistions
rlockedC := make(chan *recipe.RWMutex, 1)
wlockedC := make(chan *recipe.RWMutex, 1)
for i := 0; i < waiters; i++ {
go func() {
session, err := concurrency.NewSession(chooseClient())
if err != nil {
t.Error(err)
}
rwm := recipe.NewRWMutex(session, "test-rwmutex")
if rand.Intn(1) == 0 {
if err := rwm.RLock(); err != nil {
t.Fatalf("could not rlock (%v)", err)
}
rlockedC <- rwm
} else {
if err := rwm.Lock(); err != nil {
t.Fatalf("could not lock (%v)", err)
}
wlockedC <- rwm
}
}()
}
// unlock locked rwmutexes
timerC := time.After(time.Duration(waiters) * time.Second)
for i := 0; i < waiters; i++ {
select {
case <-timerC:
t.Fatalf("timed out waiting for lock %d", i)
case wl := <-wlockedC:
select {
case <-rlockedC:
t.Fatalf("rlock %d readers did not wait", i)
default:
}
if err := wl.Unlock(); err != nil {
t.Fatalf("could not release lock (%v)", err)
}
case rl := <-rlockedC:
select {
case <-wlockedC:
t.Fatalf("rlock %d writers did not wait", i)
default:
}
if err := rl.RUnlock(); err != nil {
t.Fatalf("could not release rlock (%v)", err)
}
}
}
}
// TestV3LeaseFailureOverlap issues Grant and Keepalive requests to a cluster
// before, during, and after quorum loss to confirm Grant/Keepalive tolerates
// transient cluster failure.
func TestV3LeaseFailureOverlap(t *testing.T) {
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 2})
defer clus.Terminate(t)
numReqs := 5
cli := clus.Client(0)
// bring up a session, tear it down
updown := func(i int) error {
sess, err := concurrency.NewSession(cli)
if err != nil {
return err
}
ch := make(chan struct{})
go func() {
defer close(ch)
sess.Close()
}()
select {
case <-ch:
case <-time.After(time.Minute / 4):
t.Fatalf("timeout %d", i)
}
return nil
}
var wg sync.WaitGroup
mkReqs := func(n int) {
wg.Add(numReqs)
for i := 0; i < numReqs; i++ {
go func() {
defer wg.Done()
err := updown(n)
if err == nil || err == rpctypes.ErrTimeoutDueToConnectionLost {
return
}
t.Fatal(err)
}()
}
}
mkReqs(1)
clus.Members[1].Stop(t)
mkReqs(2)
time.Sleep(time.Second)
mkReqs(3)
clus.Members[1].Restart(t)
mkReqs(4)
wg.Wait()
}
// TestMutexSessionRelock ensures that acquiring the same lock with the same
// session will not result in deadlock.
func TestMutexSessionRelock(t *testing.T) {
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
defer clus.Terminate(t)
session, err := concurrency.NewSession(clus.RandClient())
if err != nil {
t.Error(err)
}
m := concurrency.NewMutex(session, "test-mutex")
if err := m.Lock(context.TODO()); err != nil {
t.Fatal(err)
}
m2 := concurrency.NewMutex(session, "test-mutex")
if err := m2.Lock(context.TODO()); err != nil {
t.Fatal(err)
}
}
// TestElectionOnPrefixOfExistingKey checks that a single
// candidate can be elected on a new key that is a prefix
// of an existing key. To wit, check for regression
// of bug #6278. https://github.com/coreos/etcd/issues/6278
//
func TestElectionOnPrefixOfExistingKey(t *testing.T) {
clus := NewClusterV3(t, &ClusterConfig{Size: 1})
defer clus.Terminate(t)
cli := clus.RandClient()
if _, err := cli.Put(context.TODO(), "testa", "value"); err != nil {
t.Fatal(err)
}
s, serr := concurrency.NewSession(cli)
if serr != nil {
t.Fatal(serr)
}
e := concurrency.NewElection(s, "test")
ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second)
err := e.Campaign(ctx, "abc")
cancel()
if err != nil {
// after 5 seconds, deadlock results in
// 'context deadline exceeded' here.
t.Fatal(err)
}
}
func TestDoubleBarrier(t *testing.T) {
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
defer clus.Terminate(t)
waiters := 10
session, err := concurrency.NewSession(clus.RandClient())
if err != nil {
t.Error(err)
}
defer session.Orphan()
b := recipe.NewDoubleBarrier(session, "test-barrier", waiters)
donec := make(chan struct{})
for i := 0; i < waiters-1; i++ {
go func() {
session, err := concurrency.NewSession(clus.RandClient())
if err != nil {
t.Error(err)
}
defer session.Orphan()
bb := recipe.NewDoubleBarrier(session, "test-barrier", waiters)
if err := bb.Enter(); err != nil {
t.Fatalf("could not enter on barrier (%v)", err)
}
donec <- struct{}{}
if err := bb.Leave(); err != nil {
t.Fatalf("could not leave on barrier (%v)", err)
}
donec <- struct{}{}
}()
}
time.Sleep(10 * time.Millisecond)
select {
case <-donec:
t.Fatalf("barrier did not enter-wait")
default:
}
if err := b.Enter(); err != nil {
t.Fatalf("could not enter last barrier (%v)", err)
}
timerC := time.After(time.Duration(waiters*100) * time.Millisecond)
for i := 0; i < waiters-1; i++ {
select {
case <-timerC:
t.Fatalf("barrier enter timed out")
case <-donec:
}
}
time.Sleep(10 * time.Millisecond)
select {
case <-donec:
t.Fatalf("barrier did not leave-wait")
default:
}
b.Leave()
timerC = time.After(time.Duration(waiters*100) * time.Millisecond)
for i := 0; i < waiters-1; i++ {
select {
case <-timerC:
t.Fatalf("barrier leave timed out")
case <-donec:
}
}
}
func dropSessionLease(clus *ClusterV3) {
for _, client := range clus.clients {
s, _ := concurrency.NewSession(client)
s.Orphan()
}
}
// TestElectionWait tests if followers can correctly wait for elections.
func TestElectionWait(t *testing.T) {
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
defer clus.Terminate(t)
leaders := 3
followers := 3
var clients []*clientv3.Client
newClient := makeMultiNodeClients(t, clus.cluster, &clients)
electedc := make(chan string)
nextc := []chan struct{}{}
// wait for all elections
donec := make(chan struct{})
for i := 0; i < followers; i++ {
nextc = append(nextc, make(chan struct{}))
go func(ch chan struct{}) {
for j := 0; j < leaders; j++ {
session, err := concurrency.NewSession(newClient())
if err != nil {
t.Error(err)
}
b := concurrency.NewElection(session, "test-election")
cctx, cancel := context.WithCancel(context.TODO())
defer cancel()
s, ok := <-b.Observe(cctx)
if !ok {
t.Fatalf("could not observe election; channel closed")
}
electedc <- string(s.Kvs[0].Value)
// wait for next election round
<-ch
session.Orphan()
}
donec <- struct{}{}
}(nextc[i])
}
// elect some leaders
for i := 0; i < leaders; i++ {
go func() {
session, err := concurrency.NewSession(newClient())
if err != nil {
t.Error(err)
}
defer session.Orphan()
e := concurrency.NewElection(session, "test-election")
ev := fmt.Sprintf("electval-%v", time.Now().UnixNano())
if err := e.Campaign(context.TODO(), ev); err != nil {
t.Fatalf("failed volunteer (%v)", err)
}
// wait for followers to accept leadership
for j := 0; j < followers; j++ {
s := <-electedc
if s != ev {
t.Errorf("wrong election value got %s, wanted %s", s, ev)
}
}
// let next leader take over
if err := e.Resign(context.TODO()); err != nil {
t.Fatalf("failed resign (%v)", err)
}
// tell followers to start listening for next leader
for j := 0; j < followers; j++ {
nextc[j] <- struct{}{}
}
}()
}
// wait on followers
for i := 0; i < followers; i++ {
<-donec
}
closeClients(t, clients)
}
func runElection(eps []string, rounds int) {
rcs := make([]roundClient, 15)
validatec, releasec := make(chan struct{}, len(rcs)), make(chan struct{}, len(rcs))
for range rcs {
releasec <- struct{}{}
}
for i := range rcs {
v := fmt.Sprintf("%d", i)
observedLeader := ""
validateWaiters := 0
rcs[i].c = randClient(eps)
var (
s *concurrency.Session
err error
)
for {
s, err = concurrency.NewSession(rcs[i].c)
if err == nil {
break
}
}
e := concurrency.NewElection(s, "electors")
rcs[i].acquire = func() error {
<-releasec
ctx, cancel := context.WithCancel(context.Background())
go func() {
if ol, ok := <-e.Observe(ctx); ok {
observedLeader = string(ol.Kvs[0].Value)
if observedLeader != v {
cancel()
}
}
}()
err = e.Campaign(ctx, v)
if err == nil {
observedLeader = v
}
if observedLeader == v {
validateWaiters = len(rcs)
}
select {
case <-ctx.Done():
return nil
default:
cancel()
return err
}
}
rcs[i].validate = func() error {
if l, err := e.Leader(context.TODO()); err == nil && l != observedLeader {
return fmt.Errorf("expected leader %q, got %q", observedLeader, l)
}
validatec <- struct{}{}
return nil
}
rcs[i].release = func() error {
for validateWaiters > 0 {
select {
case <-validatec:
validateWaiters--
default:
return fmt.Errorf("waiting on followers")
}
}
if err := e.Resign(context.TODO()); err != nil {
return err
}
if observedLeader == v {
for range rcs {
releasec <- struct{}{}
}
}
observedLeader = ""
return nil
}
}
doRounds(rcs, rounds)
}