// buildExportedEndpoints builds the map to exported endpoints
func buildExportedEndpoints(conn coordclient.Connection, tenantID string, state *servicestate.ServiceState) (map[string][]export, error) {
glog.V(2).Infof("buildExportedEndpoints state: %+v", state)
result := make(map[string][]export)
for _, defep := range state.Endpoints {
if defep.Purpose == "export" {
exp := export{}
exp.vhosts = defep.VHosts
exp.endpointName = defep.Name
var err error
ep, err := buildApplicationEndpoint(state, &defep)
if err != nil {
return result, err
}
exp.endpoint = ep
key := registry.TenantEndpointKey(tenantID, exp.endpoint.Application)
if _, exists := result[key]; !exists {
result[key] = make([]export, 0)
}
result[key] = append(result[key], exp)
glog.V(2).Infof(" cached exported endpoint[%s]: %+v", key, exp)
}
}
return result, nil
}
// setImportedEndpoint sets an imported endpoint
func setImportedEndpoint(importedEndpoints *map[string]importedEndpoint, tenantID, endpointID, instanceID, virtualAddress, purpose string, port uint16) {
ie := importedEndpoint{}
ie.endpointID = endpointID
ie.virtualAddress = virtualAddress
ie.purpose = purpose
ie.instanceID = instanceID
ie.port = port
key := registry.TenantEndpointKey(tenantID, endpointID)
(*importedEndpoints)[key] = ie
glog.Infof(" cached imported endpoint[%s]: %+v", key, ie)
}
func (c *Controller) getMatchingEndpoint(id string) *importedEndpoint {
for _, ie := range c.importedEndpoints {
endpointPattern := fmt.Sprintf("^%s$", registry.TenantEndpointKey(c.tenantID, ie.endpointID))
glog.V(2).Infof(" checking tenantEndpointID %s against pattern %s", id, endpointPattern)
endpointRegex, err := regexp.Compile(endpointPattern)
if err != nil {
glog.Warningf(" unable to check tenantEndpointID %s against imported endpoint %s", id, ie.endpointID)
continue //Don't spam error message; it was reported at validation time
}
if endpointRegex.MatchString(id) {
glog.V(2).Infof(" tenantEndpointID:%s matched imported endpoint pattern:%s for %+v", id, endpointPattern, ie)
return &ie
}
}
return nil
}
func (c *Controller) handleControlCenterImports(rpcdead chan struct{}) error {
// this function is currently needed to handle special control center imports
// from GetServiceEndpoints() that does not exist in endpoints from getServiceState
// get service endpoints
client, err := node.NewLBClient(c.options.ServicedEndpoint)
if err != nil {
glog.Errorf("Could not create a client to endpoint: %s, %s", c.options.ServicedEndpoint, err)
return err
}
defer client.Close()
// TODO: instead of getting all endpoints, via GetServiceEndpoints(), create a new call
// that returns only special "controlplane" imported endpoints
// Note: GetServiceEndpoints has been modified to return only special controlplane endpoints.
// We should rename it and clean up the filtering code below.
epchan := make(chan map[string][]dao.ApplicationEndpoint)
timeout := make(chan struct{})
go func(c *node.LBClient, svcid string, epc chan map[string][]dao.ApplicationEndpoint, timeout chan struct{}) {
var endpoints map[string][]dao.ApplicationEndpoint
RetryGetServiceEndpoints:
for {
err = c.GetServiceEndpoints(svcid, &endpoints)
if err != nil {
select {
case <-time.After(1 * time.Second):
glog.V(3).Info("Couldn't retrieve service endpoints, trying again")
continue RetryGetServiceEndpoints
case <-timeout:
glog.V(3).Info("Timed out trying to retrieve service endpoints")
return
}
}
break
}
// deal with the race between the one minute timeout in handleControlCenterImports() and the
// call to GetServiceEndpoint() - the timeout may happen between GetServiceEndpoint() completing
// and sending the result via the epc channel.
select {
case _, ok := <-epc:
if ok {
panic("should never receive anything on the endpoints channel")
}
glog.V(3).Info("Endpoint channel closed, giving up")
return
default:
epc <- endpoints
}
}(client, c.options.Service.ID, epchan, timeout)
var endpoints map[string][]dao.ApplicationEndpoint
select {
case <-time.After(1 * time.Minute):
close(epchan)
timeout <- struct{}{}
client.SendLogMessage(node.ServiceLogInfo{ServiceID: c.options.Service.ID, Message: "unable to retrieve service endpoints"}, nil)
return ErrNoServiceEndpoints
case <-rpcdead:
close(epchan)
timeout <- struct{}{}
return fmt.Errorf("RPC Service has gone away")
case endpoints = <-epchan:
glog.Infof("Got service endpoints for %s: %+v", c.options.Service.ID, endpoints)
}
// convert keys set by GetServiceEndpoints to tenantID_endpointID
tmp := make(map[string][]dao.ApplicationEndpoint)
for key, endpointList := range endpoints {
if len(endpointList) <= 0 {
glog.Warningf("ignoring key: %s with empty endpointList", key)
continue
}
tenantEndpointID := registry.TenantEndpointKey(c.tenantID, endpointList[0].Application)
glog.Infof("changing key from %s to %s: %+v", key, tenantEndpointID, endpointList[0])
tmp[tenantEndpointID] = endpoints[key]
}
endpoints = tmp
cc_endpoint_purpose := "import" // Punting on control center dynamic imports for now
for key, endpointList := range endpoints {
// ignore endpoints that are not special controlplane imports
ignorePrefix := fmt.Sprintf("%s_controlplane", c.tenantID)
if !strings.HasPrefix(key, ignorePrefix) {
continue
}
// set proxy addresses
c.setProxyAddresses(key, endpointList, endpointList[0].VirtualAddress, cc_endpoint_purpose)
// add/replace entries in importedEndpoints
instanceIDStr := fmt.Sprintf("%d", endpointList[0].InstanceID)
setImportedEndpoint(&c.importedEndpoints, c.tenantID,
endpointList[0].Application, instanceIDStr,
endpointList[0].VirtualAddress, cc_endpoint_purpose,
endpointList[0].ContainerPort)
// TODO: agent needs to register controlplane and controlplane_consumer
// but don't do that here in the container code
}
return nil
}
func (dt *DaoTest) TestDao_EndpointRegistrySet(t *C) {
epr, err := registry.CreateEndpointRegistry(dt.zkConn)
t.Assert(err, IsNil)
aep := dao.ApplicationEndpoint{
ServiceID: "epn_service",
ContainerIP: "192.168.0.1",
ContainerPort: 54321,
ProxyPort: 54321,
HostIP: "192.168.0.2",
HostPort: 12345,
Protocol: "epn_tcp",
}
epn1 := registry.EndpointNode{
ApplicationEndpoint: aep,
TenantID: "epn_tenant",
EndpointID: "epn_endpoint",
HostID: "epn_host1",
ContainerID: "epn_container",
}
t.Logf("Creating a new node: %+v", epn1)
// verify that only one node gets set per host-container combo for the
// tenant-endpoint key (regardless of whether the node is ephemeral)
func(expected registry.EndpointNode) {
// case 1: add a single non-ephemeral node
epr.SetEphemeral(false)
p, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(p, Not(Equals), "")
defer dt.zkConn.Delete(path.Dir(p))
actual, err := epr.GetItem(dt.zkConn, p)
t.Assert(err, IsNil)
t.Assert(actual, NotNil)
expected.SetVersion(nil)
actual.SetVersion(nil)
t.Assert(expected, Equals, *actual)
// case 2: update the node
expected.ContainerIP = "192.168.1.1"
path2, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(path2, Equals, p)
actual, err = epr.GetItem(dt.zkConn, path2)
t.Assert(err, IsNil)
t.Assert(actual, NotNil)
expected.SetVersion(nil)
actual.SetVersion(nil)
t.Assert(expected, Equals, *actual)
// case 3: add the same node ephemerally
epr.SetEphemeral(true)
expected.ContainerIP = "192.168.2.2"
path3, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(path3, Equals, p)
actual, err = epr.GetItem(dt.zkConn, path3)
t.Assert(err, IsNil)
t.Assert(actual, NotNil)
expected.SetVersion(nil)
actual.SetVersion(nil)
t.Assert(expected, Equals, *actual)
// case 4: add a new ephemeral node
expected.ContainerID = "epn_container_E"
path4, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(path4, Not(Equals), p)
actual, err = epr.GetItem(dt.zkConn, path4)
t.Assert(err, IsNil)
t.Assert(actual, NotNil)
expected.SetVersion(nil)
actual.SetVersion(nil)
t.Assert(expected, Equals, *actual)
// case 5: add the same node non-ephemerally
/*
epr.SetEphemeral(false)
expected.ContainerIP = "192.168.3.3"
path5, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(path5, Equals, path4)
actual, err = epr.GetItem(dt.zkConn, path5)
t.Assert(err, IsNil)
t.Assert(actual, NotNil)
expected.SetVersion(nil)
actual.SetVersion(nil)
t.Assert(expected, Equals, *actual)
*/
}(epn1)
t.Logf("Testing removal of endpoint node %+v", epn1)
func(expected registry.EndpointNode) {
// case 1: remove non-ephemeral node
epr.SetEphemeral(false)
p, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(p, Not(Equals), "")
defer dt.zkConn.Delete(path.Dir(p))
err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
t.Assert(err, IsNil)
exists, _ := dt.zkConn.Exists(p)
t.Assert(exists, Equals, false)
// case 2: remove non-ephemeral node as ephemeral
/*
p, err = epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(p, Not(Equals), "")
epr.SetEphemeral(true)
err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
if err != nil {
defer dt.zkConn.Delete(p)
}
t.Assert(err, IsNil)
exists, _ = dt.zkConn.Exists(p)
t.Assert(exists, Equals, false)
*/
// case 3: remove ephemeral node
p, err = epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(p, Not(Equals), "")
err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
t.Assert(err, IsNil)
exists, _ = dt.zkConn.Exists(p)
t.Assert(exists, Equals, false)
// case 4: remove ephemeral node as non-ephemeral
p, err = epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
t.Assert(p, Not(Equals), "")
epr.SetEphemeral(false)
err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
if err != nil {
defer dt.zkConn.Delete(p)
}
t.Assert(err, IsNil)
exists, _ = dt.zkConn.Exists(p)
t.Assert(exists, Equals, false)
}(epn1)
t.Logf("Testing endpoint watcher")
func(expected registry.EndpointNode) {
errC := make(chan error)
eventC := make(chan int)
// case 0: no parent
go func() {
parentKey := registry.TenantEndpointKey("bad_tenant", "bad_endpoint")
errC <- epr.WatchTenantEndpoint(dt.zkConn, parentKey, nil, nil, make(chan bool))
}()
select {
case err := <-errC:
t.Assert(err, Equals, client.ErrNoNode)
case <-time.After(5 * time.Second):
t.Fatalf("Timeout from WatchTenantEndpoint")
// TODO: cancel listener here
}
t.Logf("Starting endpoint listener")
go func() {
changeEvt := func(conn client.Connection, parent string, nodeIDs ...string) {
eventC <- len(nodeIDs)
}
errEvt := func(path string, err error) {
errC <- err
}
parentKey := registry.TenantEndpointKey(expected.TenantID, expected.EndpointID)
_, err := epr.EnsureKey(dt.zkConn, parentKey)
t.Assert(err, IsNil)
epr.WatchTenantEndpoint(dt.zkConn, parentKey, changeEvt, errEvt, make(chan bool))
close(errC)
}()
select {
case count := <-eventC:
t.Assert(count, Equals, 0)
case err := <-errC:
t.Fatalf("unexpected error running endpoint listener: %s", err)
case <-time.After(5 * time.Second):
t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
// TODO: cancel listener here
}
// case 1: add item
p, err := epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
select {
case count := <-eventC:
t.Assert(count, Equals, 1)
case err := <-errC:
t.Fatalf("unexpected error running endpoint listener: %s", err)
case <-time.After(5 * time.Second):
t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
// TODO: cancel listener here
}
// case 2: update an item (should not receieve an event)
expected.ContainerIP = "192.168.23.12"
_, err = epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
// case 3: add another item
expected.ContainerID = "test_container_2"
_, err = epr.SetItem(dt.zkConn, expected)
t.Assert(err, IsNil)
select {
case count := <-eventC:
t.Assert(count, Equals, 2)
case err := <-errC:
t.Fatalf("unexpected error running endpoint listener: %s", err)
case <-time.After(5 * time.Second):
t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
// TODO: cancel listener here
}
// case 4: remove item
err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
t.Assert(err, IsNil)
select {
case count := <-eventC:
t.Assert(count, Equals, 1)
case err := <-errC:
t.Fatalf("unexpected error running endpoint listener: %s", err)
case <-time.After(5 * time.Second):
t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
// TODO: cancel listener here
}
// case 5: remove parent
err = dt.zkConn.Delete(path.Dir(p))
t.Assert(err, IsNil)
select {
case count := <-eventC:
t.Assert(count, Equals, 0)
case err := <-errC:
t.Assert(err, Equals, client.ErrNoNode)
case <-time.After(5 * time.Second):
t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
// TODO: cancel listener here
}
}(epn1)
}