func watch(conn client.Connection, path string, cancel <-chan bool, processChildren ProcessChildrenFunc, errorHandler WatchError) error {
exists, err := zzk.PathExists(conn, path)
if err != nil {
return err
}
if !exists {
return client.ErrNoNode
}
for {
glog.V(1).Infof("watching children at path: %s", path)
nodeIDs, event, err := conn.ChildrenW(path)
glog.V(1).Infof("child watch for path %s returned: %#v", path, nodeIDs)
if err != nil {
glog.Errorf("Could not watch %s: %s", path, err)
defer errorHandler(path, err)
return err
}
processChildren(conn, path, nodeIDs...)
select {
case ev := <-event:
glog.V(1).Infof("watch event %+v at path: %s", ev, path)
case <-cancel:
glog.V(1).Infof("watch cancel at path: %s", path)
return nil
}
}
glog.V(1).Infof("no longer watching children at path: %s", path)
return nil
}
func (r *registryType) watchItem(conn client.Connection, path string, nodeType client.Node, cancel <-chan bool, processNode func(conn client.Connection,
node client.Node), errorHandler WatchError) error {
exists, err := zzk.PathExists(conn, path)
if err != nil {
return err
}
if !exists {
return client.ErrNoNode
}
for {
event, err := conn.GetW(path, nodeType)
if err != nil {
glog.Errorf("Could not watch %s: %s", path, err)
defer errorHandler(path, err)
return err
}
processNode(conn, nodeType)
//This blocks until a change happens under the key
select {
case ev := <-event:
glog.V(2).Infof("watch event %+v at path: %s", ev, path)
case <-cancel:
return nil
}
}
return nil
}
//Set node to the key in registry. Returns the path of the node in the registry
func (r *registryType) setItem(conn client.Connection, key string, nodeID string, node client.Node) (string, error) {
if err := r.ensureKey(conn, key); err != nil {
return "", err
}
//TODO: make ephemeral
path := r.getPath(key, nodeID)
exists, err := zzk.PathExists(conn, path)
if err != nil {
return "", err
}
if exists {
glog.V(3).Infof("Set to %s: %#v", path, node)
epn := EndpointNode{}
if err := conn.Get(path, &epn); err != nil {
return "", err
}
node.SetVersion(epn.Version())
if err := conn.Set(path, node); err != nil {
return "", err
}
} else {
if addPath, err := r.addItem(conn, key, nodeID, node); err != nil {
return "", err
} else {
path = addPath
}
glog.V(3).Infof("Add to %s: %#v", path, node)
}
return path, nil
}
// LoadRunningServicesByHost returns a slice of RunningServices given a host(s)
func LoadRunningServicesByHost(conn client.Connection, hostIDs ...string) ([]dao.RunningService, error) {
var rss []dao.RunningService = make([]dao.RunningService, 0)
for _, hostID := range hostIDs {
if exists, err := zzk.PathExists(conn, hostpath(hostID)); err != nil {
return nil, err
} else if !exists {
continue
}
stateIDs, err := conn.Children(hostpath(hostID))
if err != nil {
return nil, err
}
for _, ssID := range stateIDs {
var hs HostState
if err := conn.Get(hostpath(hostID, ssID), &hs); err != nil {
return nil, err
}
rs, err := LoadRunningService(conn, hs.ServiceID, hs.ServiceStateID)
if err != nil {
return nil, err
}
rss = append(rss, *rs)
}
}
return rss, nil
}
// CreateEndpointRegistry creates the endpoint registry and returns the EndpointRegistry type
// This is created in the leader, most other calls will just get that one
func CreateEndpointRegistry(conn client.Connection) (*EndpointRegistry, error) {
path := zkEndpointsPath()
if exists, err := zzk.PathExists(conn, path); err != nil {
return nil, err
} else if !exists {
if err := conn.CreateDir(path); err != nil {
glog.Errorf("error with CreateDir(%s) %+v", path, err)
return nil, err
}
}
return &EndpointRegistry{registryType{getPath: zkEndpointsPath, ephemeral: true}}, nil
}
// RemoveServiceVhost deletes a service vhost
func RemoveServiceVhost(conn client.Connection, serviceID, vhostname string) error {
glog.V(2).Infof("RemoveServiceVhost serviceID:%s vhostname:%s", serviceID, vhostname)
// Check if the path exists
spath := servicevhostpath(serviceID, vhostname)
if exists, err := zzk.PathExists(conn, spath); err != nil {
glog.Errorf("unable to determine whether removal path exists %s %s", spath, err)
return err
} else if !exists {
glog.Errorf("service vhost removal path does not exist %s", spath)
return nil
}
// Delete the service vhost
glog.V(2).Infof("Deleting service vhost at path:%s", spath)
return conn.Delete(spath)
}
func removeNode(conn client.Connection, path string) error {
exists, err := zzk.PathExists(conn, path)
if err != nil {
return err
}
if !exists {
return nil
}
if err := conn.Delete(path); err != nil {
glog.Errorf("Unable to delete path:%s error:%v", path, err)
return err
}
return nil
}
// RemoveService deletes a service
func RemoveService(conn client.Connection, serviceID string) error {
// Check if the path exists
if exists, err := zzk.PathExists(conn, servicepath(serviceID)); err != nil {
return err
} else if !exists {
return nil
}
// If the service has any children, do not delete
if states, err := conn.Children(servicepath(serviceID)); err != nil {
return err
} else if instances := len(states); instances > 0 {
return fmt.Errorf("service %s has %d running instances", serviceID, instances)
}
// Delete the service
return conn.Delete(servicepath(serviceID))
}
//EnsureKey ensures key path to the registry. Returns the path of the key in the registry
func (r *registryType) EnsureKey(conn client.Connection, key string) (string, error) {
path := r.getPath(key)
glog.Infof("EnsureKey key:%s path:%s", key, path)
exists, err := zzk.PathExists(conn, path)
if err != nil {
return "", err
}
if !exists {
key := &KeyNode{ID: key}
if err := conn.Create(path, key); err != nil {
return "", err
}
}
glog.Infof("EnsureKey returning path:%s", path)
return path, nil
}
// AddUpdate implements zzk.SyncHandler
func (l *KeySynchronizer) AddUpdate(_ string, node zzk.Node) (string, error) {
exists, err := zzk.PathExists(l.conn, l.registry.getPath(node.GetID()))
if err != nil {
return "", err
}
if exists {
err = l.updatekey(node.GetID())
} else {
err = l.addkey(node.GetID())
}
if err != nil {
return "", err
}
return node.GetID(), nil
}
// LoadRunningServicesByService returns a slice of RunningServices per service id(s)
func LoadRunningServicesByService(conn client.Connection, serviceIDs ...string) ([]dao.RunningService, error) {
var rss []dao.RunningService
for _, serviceID := range serviceIDs {
if exists, err := zzk.PathExists(conn, servicepath(serviceID)); err != nil {
return nil, err
} else if !exists {
continue
}
stateIDs, err := conn.Children(servicepath(serviceID))
if err != nil {
return nil, err
}
for _, ssID := range stateIDs {
rs, err := LoadRunningService(conn, serviceID, ssID)
if err != nil {
return nil, err
}
rss = append(rss, *rs)
}
}
return rss, nil
}
// LoadRunningServices gets all RunningServices
func LoadRunningServices(conn client.Connection) ([]dao.RunningService, error) {
if exists, err := zzk.PathExists(conn, servicepath()); err != nil {
return nil, err
} else if !exists {
return []dao.RunningService{}, nil
}
serviceIDs, err := conn.Children(servicepath())
if err != nil {
return nil, err
}
// filter non-unique service ids
unique := make(map[string]interface{})
ids := make([]string, 0)
for _, serviceID := range serviceIDs {
if _, ok := unique[serviceID]; !ok {
unique[serviceID] = nil
ids = append(ids, serviceID)
}
}
return LoadRunningServicesByService(conn, ids...)
}
// Spawn implements zzk.Listener
func (l *VirtualIPListener) Spawn(shutdown <-chan interface{}, ip string) {
// ensure that the retry sentinel has good initial state
if l.retry == nil {
l.retry = make(map[string]int)
}
if _, ok := l.retry[ip]; !ok {
l.retry[ip] = maxRetries
}
// Check if this ip has exceeded the number of retries for this host
if l.retry[ip] > maxRetries {
glog.Warningf("Throttling acquisition of %s for %s", ip, l.hostID)
select {
case <-time.After(waitTimeout):
case <-shutdown:
return
}
}
glog.V(2).Infof("Host %s waiting to acquire virtual ip %s", l.hostID, ip)
// Try to take lead on the path
leader := zzk.NewHostLeader(l.conn, l.hostID, "", l.GetPath(ip))
_, err := leader.TakeLead()
if err != nil {
glog.Errorf("Error while trying to acquire a lock for %s: %s", ip, err)
return
}
defer l.stopInstances(ip)
defer leader.ReleaseLead()
select {
case <-shutdown:
return
default:
}
// Check if the path still exists
if exists, err := zzk.PathExists(l.conn, l.GetPath(ip)); err != nil {
glog.Errorf("Error while checking ip %s: %s", ip, err)
return
} else if !exists {
return
}
index := l.getIndex()
for {
var vip pool.VirtualIP
event, err := l.conn.GetW(l.GetPath(ip), &VirtualIPNode{VirtualIP: &vip})
if err == client.ErrEmptyNode {
glog.Errorf("Deleting empty node for ip %s", ip)
RemoveVirtualIP(l.conn, ip)
return
} else if err != nil {
glog.Errorf("Could not load virtual ip %s: %s", ip, err)
return
}
glog.V(2).Infof("Host %s binding to %s", l.hostID, ip)
rebind, err := l.bind(&vip, index)
if err != nil {
glog.Errorf("Could not bind to virtual ip %s: %s", ip, err)
l.retry[ip]++
return
}
if l.retry[ip] > 0 {
l.retry[ip]--
}
select {
case e := <-event:
// If the virtual ip is changed, you need to update the bindings
if err := l.unbind(ip); err != nil {
glog.Errorf("Could not unbind to virtual ip %s: %s", ip, err)
return
}
if e.Type == client.EventNodeDeleted {
return
}
glog.V(4).Infof("virtual ip listener for %s receieved event: %v", ip, e)
case <-rebind:
// If the primary virtual IP is removed, all other virtual IPs on
// that subnet are removed. This is in place to restore the
// virtual IPs that were removed soley by the removal of the
// primary virtual IP.
glog.V(2).Infof("Host %s rebinding to %s", l.hostID, ip)
case <-shutdown:
if err := l.unbind(ip); err != nil {
glog.Errorf("Could not unbind to virtual ip %s: %s", ip, err)
}
return
}
}
}