// buildExportedEndpoints builds the map to exported endpoints func buildExportedEndpoints(conn coordclient.Connection, tenantID string, state *servicestate.ServiceState) (map[string][]export, error) { glog.V(2).Infof("buildExportedEndpoints state: %+v", state) result := make(map[string][]export) for _, defep := range state.Endpoints { if defep.Purpose == "export" { exp := export{} exp.vhosts = defep.VHosts exp.endpointName = defep.Name var err error ep, err := buildApplicationEndpoint(state, &defep) if err != nil { return result, err } exp.endpoint = ep key := registry.TenantEndpointKey(tenantID, exp.endpoint.Application) if _, exists := result[key]; !exists { result[key] = make([]export, 0) } result[key] = append(result[key], exp) glog.V(2).Infof(" cached exported endpoint[%s]: %+v", key, exp) } } return result, nil }
// setImportedEndpoint sets an imported endpoint func setImportedEndpoint(importedEndpoints *map[string]importedEndpoint, tenantID, endpointID, instanceID, virtualAddress, purpose string, port uint16) { ie := importedEndpoint{} ie.endpointID = endpointID ie.virtualAddress = virtualAddress ie.purpose = purpose ie.instanceID = instanceID ie.port = port key := registry.TenantEndpointKey(tenantID, endpointID) (*importedEndpoints)[key] = ie glog.Infof(" cached imported endpoint[%s]: %+v", key, ie) }
func (c *Controller) getMatchingEndpoint(id string) *importedEndpoint { for _, ie := range c.importedEndpoints { endpointPattern := fmt.Sprintf("^%s$", registry.TenantEndpointKey(c.tenantID, ie.endpointID)) glog.V(2).Infof(" checking tenantEndpointID %s against pattern %s", id, endpointPattern) endpointRegex, err := regexp.Compile(endpointPattern) if err != nil { glog.Warningf(" unable to check tenantEndpointID %s against imported endpoint %s", id, ie.endpointID) continue //Don't spam error message; it was reported at validation time } if endpointRegex.MatchString(id) { glog.V(2).Infof(" tenantEndpointID:%s matched imported endpoint pattern:%s for %+v", id, endpointPattern, ie) return &ie } } return nil }
func (c *Controller) handleControlCenterImports(rpcdead chan struct{}) error { // this function is currently needed to handle special control center imports // from GetServiceEndpoints() that does not exist in endpoints from getServiceState // get service endpoints client, err := node.NewLBClient(c.options.ServicedEndpoint) if err != nil { glog.Errorf("Could not create a client to endpoint: %s, %s", c.options.ServicedEndpoint, err) return err } defer client.Close() // TODO: instead of getting all endpoints, via GetServiceEndpoints(), create a new call // that returns only special "controlplane" imported endpoints // Note: GetServiceEndpoints has been modified to return only special controlplane endpoints. // We should rename it and clean up the filtering code below. epchan := make(chan map[string][]dao.ApplicationEndpoint) timeout := make(chan struct{}) go func(c *node.LBClient, svcid string, epc chan map[string][]dao.ApplicationEndpoint, timeout chan struct{}) { var endpoints map[string][]dao.ApplicationEndpoint RetryGetServiceEndpoints: for { err = c.GetServiceEndpoints(svcid, &endpoints) if err != nil { select { case <-time.After(1 * time.Second): glog.V(3).Info("Couldn't retrieve service endpoints, trying again") continue RetryGetServiceEndpoints case <-timeout: glog.V(3).Info("Timed out trying to retrieve service endpoints") return } } break } // deal with the race between the one minute timeout in handleControlCenterImports() and the // call to GetServiceEndpoint() - the timeout may happen between GetServiceEndpoint() completing // and sending the result via the epc channel. select { case _, ok := <-epc: if ok { panic("should never receive anything on the endpoints channel") } glog.V(3).Info("Endpoint channel closed, giving up") return default: epc <- endpoints } }(client, c.options.Service.ID, epchan, timeout) var endpoints map[string][]dao.ApplicationEndpoint select { case <-time.After(1 * time.Minute): close(epchan) timeout <- struct{}{} client.SendLogMessage(node.ServiceLogInfo{ServiceID: c.options.Service.ID, Message: "unable to retrieve service endpoints"}, nil) return ErrNoServiceEndpoints case <-rpcdead: close(epchan) timeout <- struct{}{} return fmt.Errorf("RPC Service has gone away") case endpoints = <-epchan: glog.Infof("Got service endpoints for %s: %+v", c.options.Service.ID, endpoints) } // convert keys set by GetServiceEndpoints to tenantID_endpointID tmp := make(map[string][]dao.ApplicationEndpoint) for key, endpointList := range endpoints { if len(endpointList) <= 0 { glog.Warningf("ignoring key: %s with empty endpointList", key) continue } tenantEndpointID := registry.TenantEndpointKey(c.tenantID, endpointList[0].Application) glog.Infof("changing key from %s to %s: %+v", key, tenantEndpointID, endpointList[0]) tmp[tenantEndpointID] = endpoints[key] } endpoints = tmp cc_endpoint_purpose := "import" // Punting on control center dynamic imports for now for key, endpointList := range endpoints { // ignore endpoints that are not special controlplane imports ignorePrefix := fmt.Sprintf("%s_controlplane", c.tenantID) if !strings.HasPrefix(key, ignorePrefix) { continue } // set proxy addresses c.setProxyAddresses(key, endpointList, endpointList[0].VirtualAddress, cc_endpoint_purpose) // add/replace entries in importedEndpoints instanceIDStr := fmt.Sprintf("%d", endpointList[0].InstanceID) setImportedEndpoint(&c.importedEndpoints, c.tenantID, endpointList[0].Application, instanceIDStr, endpointList[0].VirtualAddress, cc_endpoint_purpose, endpointList[0].ContainerPort) // TODO: agent needs to register controlplane and controlplane_consumer // but don't do that here in the container code } return nil }
func (dt *DaoTest) TestDao_EndpointRegistrySet(t *C) { epr, err := registry.CreateEndpointRegistry(dt.zkConn) t.Assert(err, IsNil) aep := dao.ApplicationEndpoint{ ServiceID: "epn_service", ContainerIP: "192.168.0.1", ContainerPort: 54321, ProxyPort: 54321, HostIP: "192.168.0.2", HostPort: 12345, Protocol: "epn_tcp", } epn1 := registry.EndpointNode{ ApplicationEndpoint: aep, TenantID: "epn_tenant", EndpointID: "epn_endpoint", HostID: "epn_host1", ContainerID: "epn_container", } t.Logf("Creating a new node: %+v", epn1) // verify that only one node gets set per host-container combo for the // tenant-endpoint key (regardless of whether the node is ephemeral) func(expected registry.EndpointNode) { // case 1: add a single non-ephemeral node epr.SetEphemeral(false) p, err := epr.SetItem(dt.zkConn, expected) t.Assert(err, IsNil) t.Assert(p, Not(Equals), "") defer dt.zkConn.Delete(path.Dir(p)) actual, err := epr.GetItem(dt.zkConn, p) t.Assert(err, IsNil) t.Assert(actual, NotNil) expected.SetVersion(nil) actual.SetVersion(nil) t.Assert(expected, Equals, *actual) // case 2: update the node expected.ContainerIP = "192.168.1.1" path2, err := epr.SetItem(dt.zkConn, expected) t.Assert(err, IsNil) t.Assert(path2, Equals, p) actual, err = epr.GetItem(dt.zkConn, path2) t.Assert(err, IsNil) t.Assert(actual, NotNil) expected.SetVersion(nil) actual.SetVersion(nil) t.Assert(expected, Equals, *actual) // case 3: add the same node ephemerally epr.SetEphemeral(true) expected.ContainerIP = "192.168.2.2" path3, err := epr.SetItem(dt.zkConn, expected) t.Assert(err, IsNil) t.Assert(path3, Equals, p) actual, err = epr.GetItem(dt.zkConn, path3) t.Assert(err, IsNil) t.Assert(actual, NotNil) expected.SetVersion(nil) actual.SetVersion(nil) t.Assert(expected, Equals, *actual) // case 4: add a new ephemeral node expected.ContainerID = "epn_container_E" path4, err := epr.SetItem(dt.zkConn, expected) t.Assert(err, IsNil) t.Assert(path4, Not(Equals), p) actual, err = epr.GetItem(dt.zkConn, path4) t.Assert(err, IsNil) t.Assert(actual, NotNil) expected.SetVersion(nil) actual.SetVersion(nil) t.Assert(expected, Equals, *actual) // case 5: add the same node non-ephemerally /* epr.SetEphemeral(false) expected.ContainerIP = "192.168.3.3" path5, err := epr.SetItem(dt.zkConn, expected) t.Assert(err, IsNil) t.Assert(path5, Equals, path4) actual, err = epr.GetItem(dt.zkConn, path5) t.Assert(err, IsNil) t.Assert(actual, NotNil) expected.SetVersion(nil) actual.SetVersion(nil) t.Assert(expected, Equals, *actual) */ }(epn1) t.Logf("Testing removal of endpoint node %+v", epn1) func(expected registry.EndpointNode) { // case 1: remove non-ephemeral node epr.SetEphemeral(false) p, err := epr.SetItem(dt.zkConn, expected) t.Assert(err, IsNil) t.Assert(p, Not(Equals), "") defer dt.zkConn.Delete(path.Dir(p)) err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID) t.Assert(err, IsNil) exists, _ := dt.zkConn.Exists(p) t.Assert(exists, Equals, false) // case 2: remove non-ephemeral node as ephemeral /* p, err = epr.SetItem(dt.zkConn, expected) t.Assert(err, IsNil) t.Assert(p, Not(Equals), "") epr.SetEphemeral(true) err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID) if err != nil { defer dt.zkConn.Delete(p) } t.Assert(err, IsNil) exists, _ = dt.zkConn.Exists(p) t.Assert(exists, Equals, false) */ // case 3: remove ephemeral node p, err = epr.SetItem(dt.zkConn, expected) t.Assert(err, IsNil) t.Assert(p, Not(Equals), "") err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID) t.Assert(err, IsNil) exists, _ = dt.zkConn.Exists(p) t.Assert(exists, Equals, false) // case 4: remove ephemeral node as non-ephemeral p, err = epr.SetItem(dt.zkConn, expected) t.Assert(err, IsNil) t.Assert(p, Not(Equals), "") epr.SetEphemeral(false) err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID) if err != nil { defer dt.zkConn.Delete(p) } t.Assert(err, IsNil) exists, _ = dt.zkConn.Exists(p) t.Assert(exists, Equals, false) }(epn1) t.Logf("Testing endpoint watcher") func(expected registry.EndpointNode) { errC := make(chan error) eventC := make(chan int) // case 0: no parent go func() { parentKey := registry.TenantEndpointKey("bad_tenant", "bad_endpoint") errC <- epr.WatchTenantEndpoint(dt.zkConn, parentKey, nil, nil, make(chan bool)) }() select { case err := <-errC: t.Assert(err, Equals, client.ErrNoNode) case <-time.After(5 * time.Second): t.Fatalf("Timeout from WatchTenantEndpoint") // TODO: cancel listener here } t.Logf("Starting endpoint listener") go func() { changeEvt := func(conn client.Connection, parent string, nodeIDs ...string) { eventC <- len(nodeIDs) } errEvt := func(path string, err error) { errC <- err } parentKey := registry.TenantEndpointKey(expected.TenantID, expected.EndpointID) _, err := epr.EnsureKey(dt.zkConn, parentKey) t.Assert(err, IsNil) epr.WatchTenantEndpoint(dt.zkConn, parentKey, changeEvt, errEvt, make(chan bool)) close(errC) }() select { case count := <-eventC: t.Assert(count, Equals, 0) case err := <-errC: t.Fatalf("unexpected error running endpoint listener: %s", err) case <-time.After(5 * time.Second): t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected) // TODO: cancel listener here } // case 1: add item p, err := epr.SetItem(dt.zkConn, expected) t.Assert(err, IsNil) select { case count := <-eventC: t.Assert(count, Equals, 1) case err := <-errC: t.Fatalf("unexpected error running endpoint listener: %s", err) case <-time.After(5 * time.Second): t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected) // TODO: cancel listener here } // case 2: update an item (should not receieve an event) expected.ContainerIP = "192.168.23.12" _, err = epr.SetItem(dt.zkConn, expected) t.Assert(err, IsNil) // case 3: add another item expected.ContainerID = "test_container_2" _, err = epr.SetItem(dt.zkConn, expected) t.Assert(err, IsNil) select { case count := <-eventC: t.Assert(count, Equals, 2) case err := <-errC: t.Fatalf("unexpected error running endpoint listener: %s", err) case <-time.After(5 * time.Second): t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected) // TODO: cancel listener here } // case 4: remove item err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID) t.Assert(err, IsNil) select { case count := <-eventC: t.Assert(count, Equals, 1) case err := <-errC: t.Fatalf("unexpected error running endpoint listener: %s", err) case <-time.After(5 * time.Second): t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected) // TODO: cancel listener here } // case 5: remove parent err = dt.zkConn.Delete(path.Dir(p)) t.Assert(err, IsNil) select { case count := <-eventC: t.Assert(count, Equals, 0) case err := <-errC: t.Assert(err, Equals, client.ErrNoNode) case <-time.After(5 * time.Second): t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected) // TODO: cancel listener here } }(epn1) }