func (dt *DaoTest) TestDao_EndpointRegistryCreate(t *C) {
_, err := registry.CreateEndpointRegistry(dt.zkConn)
t.Assert(err, IsNil)
//test idempotence
_, err = registry.CreateEndpointRegistry(dt.zkConn)
t.Assert(err, IsNil)
}
// processTenantEndpoint updates the addresses for an imported endpoint
func (c *Controller) processTenantEndpoint(conn coordclient.Connection, parentPath string, hostContainerIDs ...string) {
glog.V(2).Infof("processTenantEndpoint: parentPath:%s hostContainerIDs: %v", parentPath, hostContainerIDs)
// update the proxy for this tenant endpoint
endpointRegistry, err := registry.CreateEndpointRegistry(conn)
if err != nil {
glog.Errorf("Could not get EndpointRegistry. Endpoints not registered: %v", err)
return
}
parts := strings.Split(parentPath, "/")
tenantEndpointID := parts[len(parts)-1]
if ep := c.getMatchingEndpoint(tenantEndpointID); ep != nil {
endpoints := make([]dao.ApplicationEndpoint, len(hostContainerIDs))
for ii, hostContainerID := range hostContainerIDs {
path := fmt.Sprintf("%s/%s", parentPath, hostContainerID)
endpointNode, err := endpointRegistry.GetItem(conn, path)
if err != nil {
glog.Errorf("error getting endpoint node at %s: %v", path, err)
continue
}
endpoints[ii] = endpointNode.ApplicationEndpoint
if ep.port != 0 {
glog.V(2).Infof("overriding ProxyPort with imported port:%v for endpoint: %+v", ep.port, endpointNode)
endpoints[ii].ProxyPort = ep.port
} else {
glog.V(2).Infof("not overriding ProxyPort with imported port:%v for endpoint: %+v", ep.port, endpointNode)
endpoints[ii].ProxyPort = endpoints[ii].ContainerPort
}
}
c.setProxyAddresses(tenantEndpointID, endpoints, ep.virtualAddress, ep.purpose)
}
}
func (c *Controller) watchregistry() <-chan struct{} {
alert := make(chan struct{}, 1)
go func() {
paths := append(c.vhostZKPaths, c.exportedEndpointZKPaths...)
if len(paths) == 0 {
return
}
conn, err := zzk.GetLocalConnection("/")
if err != nil {
return
}
endpointRegistry, err := registry.CreateEndpointRegistry(conn)
if err != nil {
glog.Errorf("Could not get EndpointRegistry. Endpoints not checked: %v", err)
return
}
interval := time.Tick(60 * time.Second)
defer func() { alert <- struct{}{} }()
for {
select {
case <-interval:
for _, path := range paths {
if _, err := endpointRegistry.GetItem(conn, path); err != nil {
glog.Errorf("Could not get endpoint. %v", err)
return
}
}
}
}
}()
return alert
}
func (dt *DaoTest) TestDao_EndpointRegistrySet(t *C) {
epr, err := registry.CreateEndpointRegistry(dt.zkConn)
t.Assert(err, IsNil)
aep := dao.ApplicationEndpoint{
ServiceID: "epn_service",
ContainerIP: "192.168.0.1",
ContainerPort: 54321,
ProxyPort: 54321,
HostIP: "192.168.0.2",
HostPort: 12345,
Protocol: "epn_tcp",
}
epn1 := registry.EndpointNode{
ApplicationEndpoint: aep,
TenantID: "epn_tenant",
EndpointID: "epn_endpoint",
HostID: "epn_host1",
ContainerID: "epn_container",
}
t.Logf("Creating a new node: %+v", epn1)
// verify that only one node gets set per host-container combo for the
// tenant-endpoint key (regardless of whether the node is ephemeral)
func(expected registry.EndpointNode) {
// case 1: add a single non-ephemeral node
epr.SetEphemeral(false)
p, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(p, Not(Equals), "")
defer dt.zkConn.Delete(path.Dir(p))
actual, err := epr.GetItem(dt.zkConn, p)
t.Assert(err, IsNil)
t.Assert(actual, NotNil)
expected.SetVersion(nil)
actual.SetVersion(nil)
t.Assert(expected, Equals, *actual)
// case 2: update the node
expected.ContainerIP = "192.168.1.1"
path2, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(path2, Equals, p)
actual, err = epr.GetItem(dt.zkConn, path2)
t.Assert(err, IsNil)
t.Assert(actual, NotNil)
expected.SetVersion(nil)
actual.SetVersion(nil)
t.Assert(expected, Equals, *actual)
// case 3: add the same node ephemerally
epr.SetEphemeral(true)
expected.ContainerIP = "192.168.2.2"
path3, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(path3, Equals, p)
actual, err = epr.GetItem(dt.zkConn, path3)
t.Assert(err, IsNil)
t.Assert(actual, NotNil)
expected.SetVersion(nil)
actual.SetVersion(nil)
t.Assert(expected, Equals, *actual)
// case 4: add a new ephemeral node
expected.ContainerID = "epn_container_E"
path4, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(path4, Not(Equals), p)
actual, err = epr.GetItem(dt.zkConn, path4)
t.Assert(err, IsNil)
t.Assert(actual, NotNil)
expected.SetVersion(nil)
actual.SetVersion(nil)
t.Assert(expected, Equals, *actual)
// case 5: add the same node non-ephemerally
/*
epr.SetEphemeral(false)
expected.ContainerIP = "192.168.3.3"
path5, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(path5, Equals, path4)
actual, err = epr.GetItem(dt.zkConn, path5)
t.Assert(err, IsNil)
t.Assert(actual, NotNil)
expected.SetVersion(nil)
actual.SetVersion(nil)
t.Assert(expected, Equals, *actual)
*/
}(epn1)
t.Logf("Testing removal of endpoint node %+v", epn1)
func(expected registry.EndpointNode) {
// case 1: remove non-ephemeral node
epr.SetEphemeral(false)
p, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(p, Not(Equals), "")
defer dt.zkConn.Delete(path.Dir(p))
err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
t.Assert(err, IsNil)
exists, _ := dt.zkConn.Exists(p)
t.Assert(exists, Equals, false)
// case 2: remove non-ephemeral node as ephemeral
/*
p, err = epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(p, Not(Equals), "")
epr.SetEphemeral(true)
err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
if err != nil {
defer dt.zkConn.Delete(p)
}
t.Assert(err, IsNil)
exists, _ = dt.zkConn.Exists(p)
t.Assert(exists, Equals, false)
*/
// case 3: remove ephemeral node
p, err = epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(p, Not(Equals), "")
err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
t.Assert(err, IsNil)
exists, _ = dt.zkConn.Exists(p)
t.Assert(exists, Equals, false)
// case 4: remove ephemeral node as non-ephemeral
p, err = epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(p, Not(Equals), "")
epr.SetEphemeral(false)
err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
if err != nil {
defer dt.zkConn.Delete(p)
}
t.Assert(err, IsNil)
exists, _ = dt.zkConn.Exists(p)
t.Assert(exists, Equals, false)
}(epn1)
t.Logf("Testing endpoint watcher")
func(expected registry.EndpointNode) {
errC := make(chan error)
eventC := make(chan int)
// case 0: no parent
go func() {
parentKey := registry.TenantEndpointKey("bad_tenant", "bad_endpoint")
errC <- epr.WatchTenantEndpoint(dt.zkConn, parentKey, nil, nil, make(chan bool))
}()
select {
case err := <-errC:
t.Assert(err, Equals, client.ErrNoNode)
case <-time.After(5 * time.Second):
t.Fatalf("Timeout from WatchTenantEndpoint")
// TODO: cancel listener here
}
t.Logf("Starting endpoint listener")
go func() {
changeEvt := func(conn client.Connection, parent string, nodeIDs ...string) {
eventC <- len(nodeIDs)
}
errEvt := func(path string, err error) {
errC <- err
}
parentKey := registry.TenantEndpointKey(expected.TenantID, expected.EndpointID)
_, err := epr.EnsureKey(dt.zkConn, parentKey)
t.Assert(err, IsNil)
epr.WatchTenantEndpoint(dt.zkConn, parentKey, changeEvt, errEvt, make(chan bool))
close(errC)
}()
select {
case count := <-eventC:
t.Assert(count, Equals, 0)
case err := <-errC:
t.Fatalf("unexpected error running endpoint listener: %s", err)
case <-time.After(5 * time.Second):
t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
// TODO: cancel listener here
}
// case 1: add item
p, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
select {
case count := <-eventC:
t.Assert(count, Equals, 1)
case err := <-errC:
t.Fatalf("unexpected error running endpoint listener: %s", err)
case <-time.After(5 * time.Second):
t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
// TODO: cancel listener here
}
// case 2: update an item (should not receieve an event)
expected.ContainerIP = "192.168.23.12"
_, err = epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
// case 3: add another item
expected.ContainerID = "test_container_2"
_, err = epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
select {
case count := <-eventC:
t.Assert(count, Equals, 2)
case err := <-errC:
t.Fatalf("unexpected error running endpoint listener: %s", err)
case <-time.After(5 * time.Second):
t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
// TODO: cancel listener here
}
// case 4: remove item
err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
t.Assert(err, IsNil)
select {
case count := <-eventC:
t.Assert(count, Equals, 1)
case err := <-errC:
t.Fatalf("unexpected error running endpoint listener: %s", err)
case <-time.After(5 * time.Second):
t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
// TODO: cancel listener here
}
// case 5: remove parent
err = dt.zkConn.Delete(path.Dir(p))
t.Assert(err, IsNil)
select {
case count := <-eventC:
t.Assert(count, Equals, 0)
case err := <-errC:
t.Assert(err, Equals, client.ErrNoNode)
case <-time.After(5 * time.Second):
t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
// TODO: cancel listener here
}
}(epn1)
}
// registerExportedEndpoints registers exported ApplicationEndpoints with zookeeper
func (c *Controller) registerExportedEndpoints() error {
// TODO: accumulate the errors so that some endpoints get registered
conn, err := zzk.GetLocalConnection("/")
if err != nil {
return err
}
endpointRegistry, err := registry.CreateEndpointRegistry(conn)
if err != nil {
glog.Errorf("Could not get EndpointRegistry. Endpoints not registered: %v", err)
return err
}
var vhostRegistry *registry.VhostRegistry
vhostRegistry, err = registry.VHostRegistry(conn)
if err != nil {
glog.Errorf("Could not get vhost registy. Endpoints not registered: %v", err)
return err
}
c.vhostZKPaths = []string{}
c.exportedEndpointZKPaths = []string{}
// register exported endpoints
for key, exportList := range c.exportedEndpoints {
for _, export := range exportList {
endpoint := export.endpoint
for _, vhost := range export.vhosts {
epName := fmt.Sprintf("%s_%v", export.endpointName, export.endpoint.InstanceID)
//delete any existing vhost that hasn't been cleaned up
vhostEndpoint := registry.NewVhostEndpoint(epName, endpoint)
if paths, err := vhostRegistry.GetChildren(conn, vhost); err != nil {
glog.V(1).Infof("error trying to clean out previous vhosts", err)
} else {
glog.V(1).Infof("cleaning vhost paths %v", paths)
//clean paths
for _, path := range paths {
if vep, err := vhostRegistry.GetItem(conn, path); err != nil {
glog.V(1).Infof("Could not read %s", path)
} else {
glog.V(4).Infof("checking instance id of %#v equal %v", vep, c.options.Service.InstanceID)
if strconv.Itoa(vep.InstanceID) == c.options.Service.InstanceID {
glog.V(1).Infof("Deleting stale vhost registration for %v at %v ", vhost, path)
conn.Delete(path)
}
}
}
}
// TODO: avoid set if item already exist with data we want
var path string
if path, err = vhostRegistry.SetItem(conn, vhost, vhostEndpoint); err != nil {
glog.Errorf("could not register vhost %s for %s: %v", vhost, epName, err)
return err
} else {
glog.Infof("Registered vhost %s for %s at %s", vhost, epName, path)
c.vhostZKPaths = append(c.vhostZKPaths, path)
}
}
//delete any exisiting endpoint that hasn't been cleaned up
if paths, err := endpointRegistry.GetChildren(conn, c.tenantID, export.endpoint.Application); err != nil {
glog.V(1).Infof("error trying to clean previous endpoints: %s", err)
} else {
glog.V(1).Infof("cleaning endpoint paths %v", paths)
//clean paths
for _, path := range paths {
if epn, err := endpointRegistry.GetItem(conn, path); err != nil {
glog.V(1).Infof("Could not read %s", path)
} else {
glog.V(4).Infof("checking instance id of %#v equal %v", epn, c.options.Service.InstanceID)
if strconv.Itoa(epn.InstanceID) == c.options.Service.InstanceID {
glog.V(1).Infof("Deleting stale endpoint registration for %v at %v ", export.endpointName, path)
conn.Delete(path)
}
}
}
}
glog.Infof("Registering exported endpoint[%s]: %+v", key, endpoint)
path, err := endpointRegistry.SetItem(conn, registry.NewEndpointNode(c.tenantID, export.endpoint.Application, c.hostID, c.dockerID, endpoint))
if err != nil {
glog.Errorf(" unable to add endpoint: %+v %v", endpoint, err)
return err
}
c.exportedEndpointZKPaths = append(c.exportedEndpointZKPaths, path)
glog.V(1).Infof(" endpoint successfully added to path: %s", path)
}
}
return nil
}
// watchRemotePorts watches imported endpoints and updates proxies
func (c *Controller) watchRemotePorts() {
/*
watch each tenant endpoint
- when endpoints are added, add the endpoint proxy if not already added
- when endpoints are added, add watch on that endpoint for updates
- when endpoints are deleted, tell that endpoint proxy to stop proxying - done with ephemeral znodes
- when endpoints are deleted, may not need to deal with removing watch on that endpoint since that watch will block forever
- deal with import regexes, i.e mysql_.*
- may not need to initially deal with removal of tenant endpoint
*/
cMuxPort = uint16(c.options.Mux.Port)
cMuxTLS = c.options.Mux.TLS
for key, endpoint := range c.importedEndpoints {
glog.V(2).Infof("importedEndpoints[%s]: %+v", key, endpoint)
}
var err error
c.zkInfo, err = getAgentZkInfo(c.options.ServicedEndpoint)
if err != nil {
glog.Errorf("Invalid zk info: %v", err)
return
}
zkConn, err := zzk.GetLocalConnection("/")
if err != nil {
glog.Errorf("watchRemotePorts - error getting zk connection: %v", err)
return
}
endpointRegistry, err := registry.CreateEndpointRegistry(zkConn)
if err != nil {
glog.Errorf("watchRemotePorts - error getting vhost registry: %v", err)
return
}
//translate closing call to endpoint cancel
cancelEndpointWatch := make(chan bool)
go func() {
select {
case errc := <-c.closing:
glog.Infof("Closing endpoint watchers")
select {
case endpointsWatchCanceller <- true:
default:
}
close(cancelEndpointWatch)
errc <- nil
}
}()
processTenantEndpoints := func(conn coordclient.Connection, parentPath string, tenantEndpointIDs ...string) {
glog.V(2).Infof("processTenantEndpoints for path: %s tenantEndpointIDs: %s", parentPath, tenantEndpointIDs)
// cancel watcher on top level /endpoints if all watchers on imported endpoints have been set up
{
ignorePrefix := fmt.Sprintf("%s_controlplane", c.tenantID)
missingWatchers := false
for id := range c.importedEndpoints {
if strings.HasPrefix(id, ignorePrefix) {
// ignore controlplane special imports for now - handleRemotePorts starts proxies for those right now
// TODO: register controlplane special imports in isvcs and watch for them
continue
}
if _, ok := watchers[id]; !ok {
missingWatchers = true
}
}
if !missingWatchers {
glog.V(2).Infof("all imports are being watched - cancelling watcher on /endpoints")
select {
case endpointsWatchCanceller <- true:
return
default:
return
}
}
}
// setup watchers for each imported tenant endpoint
watchTenantEndpoints := func(tenantEndpointKey string) {
glog.V(2).Infof(" watching tenantEndpointKey: %s", tenantEndpointKey)
for {
glog.Infof("Starting watch for tenantEndpointKey %s: %v", tenantEndpointKey, err)
if err := endpointRegistry.WatchTenantEndpoint(zkConn, tenantEndpointKey,
c.processTenantEndpoint, endpointWatchError, cancelEndpointWatch); err != nil {
glog.Errorf("error watching tenantEndpointKey %s: %v", tenantEndpointKey, err)
}
select {
case <-cancelEndpointWatch:
glog.Infof("Closing watch for tenantEndpointKey %s", tenantEndpointKey)
return
case <-time.After(500 * time.Millisecond): //prevent tight loop
}
}
}
for _, id := range tenantEndpointIDs {
glog.V(2).Infof("checking need to watch tenantEndpoint: %s %s", parentPath, id)
// add watchers if they don't exist for a tenantid_application
// and if tenant-endpoint is an import
if _, ok := watchers[id]; !ok {
if _, ok := c.importedEndpoints[id]; ok {
watchers[id] = true
go watchTenantEndpoints(id)
} else {
// look for imports with regexes that match each tenantEndpointID
ep := c.getMatchingEndpoint(id)
if ep != nil {
watchers[id] = true
go watchTenantEndpoints(id)
} else {
glog.V(2).Infof(" no need to add - not imported: %s %s for importedEndpoints: %+v", parentPath, id, c.importedEndpoints)
}
}
} else {
glog.V(2).Infof(" no need to add - existing watch tenantEndpoint: %s %s", parentPath, id)
}
// BEWARE: only need to deal with add, currently no need to deal with deletes
// since tenant endpoints are currently not deleted. only the hostid_containerid
// entries within tenantid_application are added/deleted
}
}
glog.V(2).Infof("watching endpointRegistry")
go endpointRegistry.WatchRegistry(zkConn, endpointsWatchCanceller, processTenantEndpoints, endpointWatchError)
}