func removeSession(cl *api.Client, id string) {
session := cl.Session()
_, err := session.Destroy(id, nil)
if err != nil {
log.Printf("Error while destroying session: %v", err)
}
}
// acquireKey tries to acquire a consul leader key. If successful we attain mastership.
func acquireKey(cl *api.Client, key string, ttl int, sessionName string) (string, *sequencer, bool, error) {
session := cl.Session()
entry := &api.SessionEntry{
TTL: strconv.Itoa(ttl) + "s", // ttl in seconds
Name: sessionName,
Behavior: api.SessionBehaviorDelete,
}
id, _, err := session.Create(entry, nil)
if err != nil {
log.Printf("Error while creating session: %v", err)
return "", nil, false, err
}
// Get a handle to the KV API
kv := cl.KV()
val := getHostname() + ":" + strconv.Itoa(os.Getpid()) // set value in 'hostname:pid' format
// PUT a new KV pair
p := &api.KVPair{
Key: key,
Session: id,
Value: []byte(val),
}
success, _, err := kv.Acquire(p, nil)
if err != nil {
log.Printf("Error while aquiring key: %v", err)
return "", nil, false, err
}
// get the sequencer
seq, err := getSequencer(kv, key)
if err != nil {
log.Printf("Error while retrieving sequencer %v", err)
return "", nil, false, err
}
return id, seq, success, nil
}
// createSession is used to create and maintain a session to Consul
func (d *DedupManager) createSession(client *consulapi.Client) {
START:
log.Printf("[INFO] (dedup) attempting to create session")
session := client.Session()
sessionCh := make(chan struct{})
ttl := fmt.Sprintf("%.6fs", float64(*d.config.TTL)/float64(time.Second))
se := &consulapi.SessionEntry{
Name: "Consul-Template de-duplication",
Behavior: "delete",
TTL: ttl,
}
id, _, err := session.Create(se, nil)
if err != nil {
log.Printf("[ERR] (dedup) failed to create session: %v", err)
goto WAIT
}
log.Printf("[INFO] (dedup) created session %s", id)
// Attempt to lock each template
for _, t := range d.templates {
d.wg.Add(1)
go d.attemptLock(client, id, sessionCh, t)
}
// Renew our session periodically
if err := session.RenewPeriodic("15s", id, nil, d.stopCh); err != nil {
log.Printf("[ERR] (dedup) failed to renew session: %v", err)
d.wg.Wait()
}
close(sessionCh)
WAIT:
select {
case <-time.After(sessionCreateRetry):
goto START
case <-d.stopCh:
return
}
}
// NewSession creates a new Consul session with delete behaviour which will ensure
// that all entries created under it are removed when the owning application exits.
func NewSession(client *api.Client, name string) (*Session, error) {
s := &Session{
client: client,
closeCh: makeShutdownCh(),
closedCh: make(chan struct{}),
}
session := client.Session()
id, _, err := session.Create(&api.SessionEntry{
Name: name,
Behavior: "delete",
LockDelay: time.Nanosecond,
TTL: "10s",
}, nil)
if err != nil {
return nil, err
}
go func() {
select {
case <-s.closeCh:
s.Close()
}
}()
go func() {
err = session.RenewPeriodic("10s", id, nil, s.closedCh)
if err != nil {
panic(err)
}
}()
s.ID = id
return s, nil
}
func registerSession(healthPort int, healthCheckName string, client *api.Client, log logging.Logger) string {
go setupHealthCheckEndpoint(healthPort, log)
checkReg := &api.AgentCheckRegistration{
Name: healthCheckName,
}
checkReg.AgentServiceCheck.HTTP = fmt.Sprintf("http://localhost:%d", healthPort)
checkReg.AgentServiceCheck.Interval = "1s"
err := client.Agent().CheckRegister(checkReg)
if err != nil {
log.Panic("Failed to register health check", err)
}
waitForHealthy(healthCheckName, client.Health(), log)
sessionEntry := &api.SessionEntry{
Checks: []string{checkReg.Name},
}
session, _, err := client.Session().Create(sessionEntry, nil)
if err != nil {
log.Panic("Unable to create session", err)
}
for {
entry, _, err := client.Session().Info(session, nil)
if err != nil {
log.Panic("Unable to read session info", err)
}
if entry != nil {
break
}
}
return session
}
func NewSessionManager(client *api.Client) *sessionMgr {
return &sessionMgr{
client: client,
session: client.Session(),
}
}
// MaybeAcquireLeadership function takes a consul client, leader key string, check interval (in seconds), session ttl (in seconds), session name, exit on lock found as well
// as a DoJob implementation. It tries to acquire a lock by associating a session to the key. If acquired, it attains mastership setting the value of
// the key to hostname:pid. The DoJobFunc implementation is run in a go routine. The function could run till it ends voluntarily closing the doneCh channel. The api could
// could sent a stop signal via the stopCh in case leadership is lost. In such a situation the DoJobFunc implementaion should return.
// The exitOnLockFound parameter should be set to true in situations where you want your application to exit if lock is found. For continuosly applications
// needing high availability support this should be set to false. The api leverages the TTL field of sessions. The following text from the consul.io is useful to know.
//
// When creating a session, a TTL can be specified. If the TTL interval expires without being renewed, the session has expired and an invalidation is triggered.
// This type of failure detector is also known as a heartbeat failure detector. It is less scalable than the gossip based failure detector as it places an
// increased burden on the servers but may be applicable in some cases. The contract of a TTL is that it represents a lower bound for invalidation; that is,
// Consul will not expire the session before the TTL is reached, but it is allowed to delay the expiration past the TTL. The TTL is renewed on session creation,
// on session renew, and on leader failover. When a TTL is being used, clients should be aware of clock skew issues: namely, time may not progress at the same
// rate on the client as on the Consul servers. It is best to set conservative TTL values and to renew in advance of the TTL to account for network delay and time skew.
//
// The final nuance is that sessions may provide a lock-delay. This is a time duration, between 0 and 60 seconds. When a session invalidation takes place,
// Consul prevents any of the previously held locks from being re-acquired for the lock-delay interval; this is a safeguard inspired by Google's Chubby.
// The purpose of this delay is to allow the potentially still live leader to detect the invalidation and stop processing requests that may lead to inconsistent state.
// While not a bulletproof method, it does avoid the need to introduce sleep states into application logic and can help mitigate many issues.
//While the default is to use a 15 second delay, clients are able to disable this mechanism by providing a zero delay value.
func MaybeAcquireLeadership(client *api.Client, leaderKey string, leadershipCheckInterval int, sessionTTL int, sessionName string, exitOnLockFound bool, j DoJob) {
l := leader{}
sleepTime := time.Duration(leadershipCheckInterval)
// buffered to accept if we receive the stop signal.
doneCh := make(chan bool, 1)
for {
select {
case <-doneCh:
log.Println("Received done signal, exiting..")
return
default:
id, seq, success, err := acquireKey(client, leaderKey, sessionTTL, sessionName)
if err != nil {
l.errorRetryCount++
goto LABEL
}
if success {
// Maybe the key/session got removed by administrator. Close the renewperiodic channel.
if l.isLeader {
close(l.stopSessionRenewCh)
}
log.Println("Consul leadership lock acquired. Assuming leadership.")
stopSessionRenewCh := make(chan struct{})
l.stopSessionRenewCh = stopSessionRenewCh
go client.Session().RenewPeriodic(strconv.Itoa(sessionTTL)+"s", id, nil, l.stopSessionRenewCh)
if !l.isLeader {
stopCh := make(chan bool)
l.stopCh = stopCh
go j.DoJobFunc(l.stopCh, doneCh)
}
l.isLeader = true
l.seq = seq
time.Sleep(sleepTime * time.Second)
continue
}
// We reached here becoz we could not acquire the key although it is possible that we are still the master.
removeSession(client, id)
if !l.isLeader && exitOnLockFound {
log.Println("Consul leadership lock already acquired by some other process. Exiting...")
os.Exit(0)
}
if l.isLeader {
log.Printf("I still hold the consul leadership lock. Checking again in %d secs.", sleepTime)
} else {
log.Printf("Consul leadership lock is already aquired by some other process. Checking again in %d secs.", sleepTime)
}
LABEL:
if l.isLeader {
if err != nil && l.errorRetryCount > errorRetryThreshold {
log.Println("I might have lost leadership. Sending stop signal..")
close(l.stopCh)
close(l.stopSessionRenewCh)
l.isLeader = false
l.seq = nil
} else if !reflect.DeepEqual(l.seq, seq) {
log.Println("Lost leadership. Sending stop signal...")
close(l.stopCh)
close(l.stopSessionRenewCh)
l.isLeader = false
l.seq = nil
}
}
time.Sleep(sleepTime * time.Second)
}
}
}
session, sessionErr = consuladapter.NewSessionNoChecks("a-session", 20*time.Second, client, sessionMgr)
} else {
session, sessionErr = consuladapter.NewSession("a-session", 20*time.Second, client, sessionMgr)
}
})
AfterEach(func() {
if session != nil {
session.Destroy()
}
})
sessionCreationTests := func(operationErr func() error) {
It("is set with the expected defaults", func() {
entries, _, err := client.Session().List(nil)
Expect(err).NotTo(HaveOccurred())
entry := findSession(session.ID(), entries)
Expect(entry).NotTo(BeNil())
Expect(entry.Name).To(Equal("a-session"))
Expect(entry.ID).To(Equal(session.ID()))
Expect(entry.Behavior).To(Equal(api.SessionBehaviorDelete))
Expect(entry.TTL).To(Equal("20s"))
Expect(entry.LockDelay).To(BeZero())
})
It("renews the session periodically", func() {
Eventually(sessionMgr.RenewPeriodicCallCount).ShouldNot(Equal(0))
})
Context("when NodeName() fails", func() {