//Acquire will attempt to acquire the lock, if blocking is set to true it will wait forever to do so.
//Setting blocking to false would be the equivalent of a fictional TryAcquire, an immediate return
//if locking fails.
func Acquire(c *etcd.Client, key, value string, ttl uint64, blocking bool) (*Lock, error) {
index := uint64(0)
var err error
tryAcquire := true
LOOP:
for {
if tryAcquire {
index, err = acquire(c, key, value, ttl)
if err == nil {
break LOOP
}
if !blocking {
return nil, err
}
tryAcquire = false
}
resp, err := c.Watch(key, 0, false, nil, nil)
if err != nil {
return nil, err
}
if resp.Action != "compareAndSwap" {
tryAcquire = true
}
}
return &Lock{
c: c,
key: key,
value: value,
ttl: ttl,
index: index,
held: true,
}, nil
}
func GetAndWatchEpoch(client *etcd.Client, appname string, epochC chan uint64, stop chan bool) (uint64, error) {
resp, err := client.Get(EpochPath(appname), false, false)
if err != nil {
log.Fatal("etcdutil: can not get epoch from etcd")
}
ep, err := strconv.ParseUint(resp.Node.Value, 10, 64)
if err != nil {
return 0, err
}
receiver := make(chan *etcd.Response, 1)
go client.Watch(EpochPath(appname), resp.EtcdIndex+1, false, receiver, stop)
go func() {
for resp := range receiver {
if resp.Action != "compareAndSwap" && resp.Action != "set" {
continue
}
epoch, err := strconv.ParseUint(resp.Node.Value, 10, 64)
if err != nil {
log.Fatal("etcdutil: can't parse epoch from etcd")
}
epochC <- epoch
}
}()
return ep, nil
}
// This will attempt to Acquire a lock for a given amount of time.
// If the lock is acquired, goChan will be unblocked
// If the lock is lost, stopChan will be unblocked
// TODO: Handle non-expected errors
func Acquire(cli *etcd.Client, lock string, timeout uint64) (goChan chan int, stopChan chan int) {
me := uuid.NewV1().String()
goChan = make(chan int)
stopChan = make(chan int)
go func() {
log.Println("Hello, I am:", me)
for {
resp, acq, err := cli.TestAndSet(lock, "", me, timeout)
if debug {
log.Println("Lock Resp:", acq, resp, err)
}
if !acq {
// We want to watch for a change in the lock, and we'll repeat
var watcherCh = make(chan *store.Response)
var endCh = make(chan bool)
go cli.Watch(lock, 0, watcherCh, endCh)
<-watcherCh
// Now, we'll try to acquire the lock, again
} else {
// We got a lock, we want to keep it
go func() {
for {
resp, acq, err := cli.TestAndSet(lock, me, me, timeout) // Keep the lock alive
if debug {
log.Println("Reset Resp:", acq, resp, err)
}
if !acq {
if debug {
log.Println("Demoted:", me)
}
stopChan <- 1 // Let's boot ourselves, we're no longer the leader
}
time.Sleep(time.Duration(timeout*500) * time.Millisecond) // We'll re-up after 50% fo the lock period
}
}()
if debug {
log.Println("King:", me)
}
goChan <- 1
}
}
}()
return
}
func (b *backends) Watch(client *etcd.Client) {
receiver := make(chan *etcd.Response)
go client.Watch(b.path, uint64(b.watchIndex), true, receiver, nil)
for {
resp := <-receiver
b.Update(resp.Node, resp.Action)
}
}
// WaitFreeTask blocks until it gets a hint of free task
func WaitFreeTask(client *etcd.Client, name string, logger *log.Logger) (uint64, error) {
slots, err := client.Get(FreeTaskDir(name), false, true)
if err != nil {
return 0, err
}
if total := len(slots.Node.Nodes); total > 0 {
ri := rand.Intn(total)
s := slots.Node.Nodes[ri]
idStr := path.Base(s.Key)
id, err := strconv.ParseUint(idStr, 0, 64)
if err != nil {
return 0, err
}
logger.Printf("got free task %v, randomly choose %d to try...", ListKeys(slots.Node.Nodes), ri)
return id, nil
}
watchIndex := slots.EtcdIndex + 1
respChan := make(chan *etcd.Response, 1)
go func() {
for {
logger.Printf("start to wait failure at index %d", watchIndex)
resp, err := client.Watch(FreeTaskDir(name), watchIndex, true, nil, nil)
if err != nil {
logger.Printf("WARN: WaitFailure watch failed: %v", err)
return
}
if resp.Action == "set" {
respChan <- resp
return
}
watchIndex = resp.EtcdIndex + 1
}
}()
var resp *etcd.Response
var waitTime uint64 = 0
for {
select {
case resp = <-respChan:
idStr := path.Base(resp.Node.Key)
id, err := strconv.ParseUint(idStr, 10, 64)
if err != nil {
return 0, err
}
return id, nil
case <-time.After(10 * time.Second):
waitTime++
logger.Printf("Node already wait failure for %d0s", waitTime)
}
}
}
// detect failure of the given taskID
func DetectFailure(client *etcd.Client, name string, stop chan bool) error {
receiver := make(chan *etcd.Response, 1)
go client.Watch(HealthyPath(name), 0, true, receiver, stop)
for resp := range receiver {
if resp.Action != "expire" && resp.Action != "delete" {
continue
}
if err := ReportFailure(client, name, path.Base(resp.Node.Key)); err != nil {
return err
}
}
return nil
}
func watchForUpdates(client *etcd.Client, key string, index uint64) {
for {
if _, err := client.Watch(key, index, false, addressConfig, nil); err != nil {
toSleep := 5 * time.Second
log.Debug("error watching etcd for key %v: %v", key, err)
log.Debug("retry in %v", toSleep)
time.Sleep(toSleep)
}
}
}
func WatchConfig(client *etcd.Client) {
watchChan := make(chan *etcd.Response)
go client.Watch(EtcdPath, 0, true, watchChan, nil)
go func() {
for {
select {
case resp := <-watchChan:
log.Printf("Config is changed, %s, %s", resp.Node.Key, resp.Node.Value)
ReloadConfig(client)
}
}
}()
}
// Subscribe continuously updates the target structure with data from etcd as it
// is changed. This is ideal for things such as configuration or a list of keys for
// authentication.
func Subscribe(e *etcd.Client, target interface{}, prefix string) (err error) {
updates := make(chan *etcd.Response, 10)
e.Watch(prefix, 0, true, updates, make(chan bool))
go func() {
for update := range updates {
_ = update // TODO: replace me with a better method
Demarshal(e, target)
}
}()
return
}
// etcdWatch calls etcd's Watch function, and handles any errors. Meant to be called
// as a goroutine.
func (w *etcdWatcher) etcdWatch(client *etcd.Client, key string, resourceVersion uint64) {
defer util.HandleCrash()
defer close(w.etcdError)
if resourceVersion == 0 {
latest, err := etcdGetInitialWatchState(client, key, w.list, w.etcdIncoming)
if err != nil {
w.etcdError <- err
return
}
resourceVersion = latest + 1
}
_, err := client.Watch(key, resourceVersion, w.list, w.etcdIncoming, w.etcdStop)
if err != nil && err != etcd.ErrWatchStoppedByUser {
w.etcdError <- err
}
}
func WatchMeta(c *etcd.Client, path string, stop chan bool, responseHandler func(*etcd.Response)) error {
resp, err := c.Get(path, false, false)
if err != nil {
return err
}
// Get previous meta. We need to handle it.
if resp.Node.Value != "" {
go responseHandler(resp)
}
receiver := make(chan *etcd.Response, 1)
go c.Watch(path, resp.EtcdIndex+1, false, receiver, stop)
go func(receiver chan *etcd.Response) {
for resp := range receiver {
responseHandler(resp)
}
}(receiver)
return nil
}
func NewCache(client *etcd.Client, path string) *Cache {
stopChan := make(chan bool)
receiverChan := make(chan *etcd.Response)
go client.Watch(path, 0, receiverChan, stopChan)
response, _ := client.Get(path)
cachedItems := map[string]*etcd.Response{}
for _, element := range response {
cachedItems[element.Key] = element
}
cache := &Cache{
client: client,
path: path,
items: cachedItems,
}
go cache.Watcher(receiverChan, stopChan)
return cache
}
func SaveLives(etcd *etcd.Client, startHandler *starter.Starter) {
since := uint64(0)
for {
change, err := etcd.Watch("/apps", since, true, nil, nil)
if err != nil {
fmt.Println(ansi.Color("watch failed; resting up", "red"))
time.Sleep(1 * time.Second)
continue
}
since = change.Node.ModifiedIndex + 1
if change.Action == "delete" || change.Action == "expire" {
go resurrect(startHandler, change.Node.Key)
}
}
}
// CountWatch watches for some number of joins. It can be used to discover
// when a known number of group memebers are present.
//
// w := condition.NewCountWatch(client, "path", "to", "watch")
// defer w.Stop()
//
// The path must be to an etcd directory, otherwise the watch will not
// function.
//
// for {
// select {
// case <- w.WatchUntil(10):
// ... 10 partners are present ...
// case err := <- w.WatchError():
// ... error code ...
// }
// }
//
// Or
// for {
// select {
// case n := <- w.WatchCount():
// ... n partners are present ...
// case err := <- w.WatchError():
// ... error code ...
// }
// }
//
// It is not correct to select from both WatchUntil and WatchCount at
// the same time.
func NewCountWatch(e *etcd.Client, path ...string) CountWatch {
key := strings.Join(path, "/")
stopc := make(chan bool)
countc := make(chan int, 1)
errorc := make(chan error, 1)
go func() {
defer e.Close()
var res *etcd.Response
var err error
var exists bool
for !exists {
res, err = e.Get(key, false, false)
if err != nil {
switch err := err.(type) {
case *etcd.EtcdError:
if err.ErrorCode == 100 {
ticker := time.NewTicker(5 * time.Second)
select {
case <-stopc:
ticker.Stop()
return
case <-ticker.C:
ticker.Stop()
}
}
default:
errorc <- err
return
}
} else {
exists = true
}
}
index := uint64(0)
if res != nil && res.Node != nil && res.Node.Dir {
select {
case countc <- len(res.Node.Nodes):
default:
}
index = res.EtcdIndex
} else {
select {
case countc <- 0:
default:
}
}
watch := make(chan *etcd.Response)
go e.Watch(key, index, true, watch, stopc)
for {
select {
case <-stopc:
return
case res, open := <-watch:
if !open {
select {
case errorc <- fmt.Errorf("count watch closed unexpectedly: %v", key):
default:
}
return
}
res, err := e.Get(key, false, false)
if err != nil {
switch err := err.(type) {
case *etcd.EtcdError:
if err.ErrorCode == 100 {
// Do nothing.
}
default:
errorc <- err
return
}
}
if res != nil && res.Node != nil && res.Node.Dir {
select {
case countc <- len(res.Node.Nodes):
default:
}
} else {
select {
case countc <- 0:
default:
}
}
}
}
}()
return &countwatch{
stopc: stopc,
countc: countc,
errorc: errorc,
}
}
// execWatchCommandFunc executes the "exec-watch" command.
func execWatchCommandFunc(cmd *cobra.Command, args []string, client *etcd.Client) (*etcd.Response, error) {
// _ = io.Copy
// _ = exec.Command
// args := c.Args()
argsLen := len(args)
if argsLen < 2 {
return nil, errors.New("Key and command to exec required")
}
// key := args[argsLen-1]
key := args[0]
cmdArgs := args[1:]
index := 0
if execAfterIndexFlag != 0 {
index = execAfterIndexFlag + 1
}
recursive := execRecursiveFlag
sigch := make(chan os.Signal, 1)
signal.Notify(sigch, os.Interrupt)
stop := make(chan bool)
go func() {
<-sigch
stop <- true
os.Exit(0)
}()
receiver := make(chan *etcd.Response)
client.SetConsistency(etcd.WEAK_CONSISTENCY)
go client.Watch(key, uint64(index), recursive, receiver, stop)
for {
resp := <-receiver
cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
cmd.Env = environResponse(resp, os.Environ())
stdout, err := cmd.StdoutPipe()
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
stderr, err := cmd.StderrPipe()
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
err = cmd.Start()
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
go io.Copy(os.Stdout, stdout)
go io.Copy(os.Stderr, stderr)
cmd.Wait()
}
return nil, nil
}
// etcd.Watch is blocking so use this helper function to monitor.
func watch(client *etcd.Client, receiver chan *etcd.Response, stop chan bool) {
if _, err := client.Watch("/registry/pods", 0, true, receiver, stop); err != nil {
log.Fatal(err)
}
}
// NameWatch watches for specific member names to join. It can be used
// to discover when a known group of members is present.
//
// w := condition.NewNameWatch(client, "path", "to", "watch")
// defer w.Stop()
//
// The path must be to an etcd directory, otherwise the watch will not
// function.
//
// for {
// select {
// case <- w.WatchUntil("member1", "member2"):
// ... 2 partners are present ...
// case err := <- w.WatchError():
// ... error code ...
// }
// }
//
// Or
// for {
// select {
// case names := <- w.WatchNames():
// ... names of partners currently present ...
// case err := <- w.WatchError():
// ... error code ...
// }
// }
//
// It is not correct to select from both WatchUntil and WatchCount at
// the same time.
func NewNameWatch(e *etcd.Client, path ...string) NameWatch {
key := strings.Join(path, "/")
stopc := make(chan bool)
namesc := make(chan []string, 1)
errorc := make(chan error, 1)
go func() {
defer e.Close()
var res *etcd.Response
var err error
var exists bool
for !exists {
res, err = e.Get(key, false, false)
if err != nil {
switch err := err.(type) {
case *etcd.EtcdError:
if err.ErrorCode == 100 {
ticker := time.NewTicker(5 * time.Second)
select {
case <-stopc:
ticker.Stop()
return
case <-ticker.C:
ticker.Stop()
}
}
default:
errorc <- err
return
}
} else {
exists = true
}
}
index := uint64(0)
if res != nil && res.Node != nil && res.Node.Dir {
names := make([]string, 0, len(res.Node.Nodes))
for _, n := range res.Node.Nodes {
names = append(names, n.Key)
}
select {
case namesc <- names:
default:
}
index = res.EtcdIndex
} else {
select {
case namesc <- nil:
default:
}
}
watch := make(chan *etcd.Response)
go e.Watch(key, index, true, watch, stopc)
for {
select {
case <-stopc:
return
case res, open := <-watch:
if !open {
select {
case errorc <- fmt.Errorf("name watch closed unexpectedly for: %v", key):
default:
}
return
}
res, err := e.Get(key, false, false)
if err != nil {
switch err := err.(type) {
case *etcd.EtcdError:
if err.ErrorCode == 100 {
// Do nothing.
}
default:
errorc <- err
return
}
}
if res != nil && res.Node != nil && res.Node.Dir {
names := make([]string, 0, len(res.Node.Nodes))
for _, n := range res.Node.Nodes {
names = append(names, n.Key)
}
select {
case namesc <- names:
default:
}
} else {
select {
case namesc <- nil:
default:
}
}
}
}
}()
return &namewatch{
stopc: stopc,
namesc: namesc,
errorc: errorc,
}
}