示例#1
0
func (dt *DaoTest) TestDao_EndpointRegistryCreate(t *C) {

	_, err := registry.CreateEndpointRegistry(dt.zkConn)
	t.Assert(err, IsNil)

	//test idempotence
	_, err = registry.CreateEndpointRegistry(dt.zkConn)
	t.Assert(err, IsNil)
}
示例#2
0
// processTenantEndpoint updates the addresses for an imported endpoint
func (c *Controller) processTenantEndpoint(conn coordclient.Connection, parentPath string, hostContainerIDs ...string) {
	glog.V(2).Infof("processTenantEndpoint: parentPath:%s hostContainerIDs: %v", parentPath, hostContainerIDs)

	// update the proxy for this tenant endpoint
	endpointRegistry, err := registry.CreateEndpointRegistry(conn)
	if err != nil {
		glog.Errorf("Could not get EndpointRegistry. Endpoints not registered: %v", err)
		return
	}

	parts := strings.Split(parentPath, "/")
	tenantEndpointID := parts[len(parts)-1]

	if ep := c.getMatchingEndpoint(tenantEndpointID); ep != nil {
		endpoints := make([]dao.ApplicationEndpoint, len(hostContainerIDs))
		for ii, hostContainerID := range hostContainerIDs {
			path := fmt.Sprintf("%s/%s", parentPath, hostContainerID)
			endpointNode, err := endpointRegistry.GetItem(conn, path)
			if err != nil {
				glog.Errorf("error getting endpoint node at %s: %v", path, err)
				continue
			}
			endpoints[ii] = endpointNode.ApplicationEndpoint
			if ep.port != 0 {
				glog.V(2).Infof("overriding ProxyPort with imported port:%v for endpoint: %+v", ep.port, endpointNode)
				endpoints[ii].ProxyPort = ep.port
			} else {
				glog.V(2).Infof("not overriding ProxyPort with imported port:%v for endpoint: %+v", ep.port, endpointNode)
				endpoints[ii].ProxyPort = endpoints[ii].ContainerPort
			}
		}
		c.setProxyAddresses(tenantEndpointID, endpoints, ep.virtualAddress, ep.purpose)
	}
}
示例#3
0
func (c *Controller) watchregistry() <-chan struct{} {
	alert := make(chan struct{}, 1)

	go func() {

		paths := append(c.vhostZKPaths, c.exportedEndpointZKPaths...)
		if len(paths) == 0 {
			return
		}

		conn, err := zzk.GetLocalConnection("/")
		if err != nil {
			return
		}

		endpointRegistry, err := registry.CreateEndpointRegistry(conn)
		if err != nil {
			glog.Errorf("Could not get EndpointRegistry. Endpoints not checked: %v", err)
			return
		}

		interval := time.Tick(60 * time.Second)

		defer func() { alert <- struct{}{} }()
		for {
			select {
			case <-interval:
				for _, path := range paths {
					if _, err := endpointRegistry.GetItem(conn, path); err != nil {
						glog.Errorf("Could not get endpoint. %v", err)
						return
					}
				}
			}
		}
	}()

	return alert
}
示例#4
0
func (dt *DaoTest) TestDao_EndpointRegistrySet(t *C) {

	epr, err := registry.CreateEndpointRegistry(dt.zkConn)
	t.Assert(err, IsNil)

	aep := dao.ApplicationEndpoint{
		ServiceID:     "epn_service",
		ContainerIP:   "192.168.0.1",
		ContainerPort: 54321,
		ProxyPort:     54321,
		HostIP:        "192.168.0.2",
		HostPort:      12345,
		Protocol:      "epn_tcp",
	}

	epn1 := registry.EndpointNode{
		ApplicationEndpoint: aep,
		TenantID:            "epn_tenant",
		EndpointID:          "epn_endpoint",
		HostID:              "epn_host1",
		ContainerID:         "epn_container",
	}
	t.Logf("Creating a new node: %+v", epn1)

	// verify that only one node gets set per host-container combo for the
	// tenant-endpoint key (regardless of whether the node is ephemeral)
	func(expected registry.EndpointNode) {
		// case 1: add a single non-ephemeral node
		epr.SetEphemeral(false)
		p, err := epr.SetItem(dt.zkConn, expected)
		t.Assert(err, IsNil)
		t.Assert(p, Not(Equals), "")
		defer dt.zkConn.Delete(path.Dir(p))

		actual, err := epr.GetItem(dt.zkConn, p)
		t.Assert(err, IsNil)
		t.Assert(actual, NotNil)

		expected.SetVersion(nil)
		actual.SetVersion(nil)
		t.Assert(expected, Equals, *actual)

		// case 2: update the node
		expected.ContainerIP = "192.168.1.1"
		path2, err := epr.SetItem(dt.zkConn, expected)
		t.Assert(err, IsNil)
		t.Assert(path2, Equals, p)

		actual, err = epr.GetItem(dt.zkConn, path2)
		t.Assert(err, IsNil)
		t.Assert(actual, NotNil)

		expected.SetVersion(nil)
		actual.SetVersion(nil)
		t.Assert(expected, Equals, *actual)

		// case 3: add the same node ephemerally
		epr.SetEphemeral(true)
		expected.ContainerIP = "192.168.2.2"
		path3, err := epr.SetItem(dt.zkConn, expected)
		t.Assert(err, IsNil)
		t.Assert(path3, Equals, p)

		actual, err = epr.GetItem(dt.zkConn, path3)
		t.Assert(err, IsNil)
		t.Assert(actual, NotNil)

		expected.SetVersion(nil)
		actual.SetVersion(nil)
		t.Assert(expected, Equals, *actual)

		// case 4: add a new ephemeral node
		expected.ContainerID = "epn_container_E"
		path4, err := epr.SetItem(dt.zkConn, expected)
		t.Assert(err, IsNil)
		t.Assert(path4, Not(Equals), p)

		actual, err = epr.GetItem(dt.zkConn, path4)
		t.Assert(err, IsNil)
		t.Assert(actual, NotNil)

		expected.SetVersion(nil)
		actual.SetVersion(nil)
		t.Assert(expected, Equals, *actual)

		// case 5: add the same node non-ephemerally
		/*
			epr.SetEphemeral(false)
			expected.ContainerIP = "192.168.3.3"
			path5, err := epr.SetItem(dt.zkConn, expected)
			t.Assert(err, IsNil)
			t.Assert(path5, Equals, path4)

			actual, err = epr.GetItem(dt.zkConn, path5)
			t.Assert(err, IsNil)
			t.Assert(actual, NotNil)

			expected.SetVersion(nil)
			actual.SetVersion(nil)
			t.Assert(expected, Equals, *actual)
		*/
	}(epn1)

	t.Logf("Testing removal of endpoint node %+v", epn1)
	func(expected registry.EndpointNode) {
		// case 1: remove non-ephemeral node
		epr.SetEphemeral(false)
		p, err := epr.SetItem(dt.zkConn, expected)
		t.Assert(err, IsNil)
		t.Assert(p, Not(Equals), "")
		defer dt.zkConn.Delete(path.Dir(p))

		err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
		t.Assert(err, IsNil)
		exists, _ := dt.zkConn.Exists(p)
		t.Assert(exists, Equals, false)

		// case 2: remove non-ephemeral node as ephemeral
		/*
			p, err = epr.SetItem(dt.zkConn, expected)
			t.Assert(err, IsNil)
			t.Assert(p, Not(Equals), "")

			epr.SetEphemeral(true)
			err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
			if err != nil {
				defer dt.zkConn.Delete(p)
			}
			t.Assert(err, IsNil)
			exists, _ = dt.zkConn.Exists(p)
			t.Assert(exists, Equals, false)
		*/

		// case 3: remove ephemeral node
		p, err = epr.SetItem(dt.zkConn, expected)
		t.Assert(err, IsNil)
		t.Assert(p, Not(Equals), "")

		err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
		t.Assert(err, IsNil)
		exists, _ = dt.zkConn.Exists(p)
		t.Assert(exists, Equals, false)

		// case 4: remove ephemeral node as non-ephemeral
		p, err = epr.SetItem(dt.zkConn, expected)
		t.Assert(err, IsNil)
		t.Assert(p, Not(Equals), "")

		epr.SetEphemeral(false)
		err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
		if err != nil {
			defer dt.zkConn.Delete(p)
		}
		t.Assert(err, IsNil)
		exists, _ = dt.zkConn.Exists(p)
		t.Assert(exists, Equals, false)
	}(epn1)

	t.Logf("Testing endpoint watcher")
	func(expected registry.EndpointNode) {
		errC := make(chan error)
		eventC := make(chan int)

		// case 0: no parent
		go func() {
			parentKey := registry.TenantEndpointKey("bad_tenant", "bad_endpoint")
			errC <- epr.WatchTenantEndpoint(dt.zkConn, parentKey, nil, nil, make(chan bool))
		}()
		select {
		case err := <-errC:
			t.Assert(err, Equals, client.ErrNoNode)
		case <-time.After(5 * time.Second):
			t.Fatalf("Timeout from WatchTenantEndpoint")
			// TODO: cancel listener here
		}

		t.Logf("Starting endpoint listener")
		go func() {
			changeEvt := func(conn client.Connection, parent string, nodeIDs ...string) {
				eventC <- len(nodeIDs)
			}

			errEvt := func(path string, err error) {
				errC <- err
			}

			parentKey := registry.TenantEndpointKey(expected.TenantID, expected.EndpointID)
			_, err := epr.EnsureKey(dt.zkConn, parentKey)
			t.Assert(err, IsNil)

			epr.WatchTenantEndpoint(dt.zkConn, parentKey, changeEvt, errEvt, make(chan bool))
			close(errC)
		}()

		select {
		case count := <-eventC:
			t.Assert(count, Equals, 0)
		case err := <-errC:
			t.Fatalf("unexpected error running endpoint listener: %s", err)
		case <-time.After(5 * time.Second):
			t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
			// TODO: cancel listener here
		}

		// case 1: add item
		p, err := epr.SetItem(dt.zkConn, expected)
		t.Assert(err, IsNil)

		select {
		case count := <-eventC:
			t.Assert(count, Equals, 1)
		case err := <-errC:
			t.Fatalf("unexpected error running endpoint listener: %s", err)
		case <-time.After(5 * time.Second):
			t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
			// TODO: cancel listener here
		}

		// case 2: update an item (should not receieve an event)
		expected.ContainerIP = "192.168.23.12"
		_, err = epr.SetItem(dt.zkConn, expected)
		t.Assert(err, IsNil)

		// case 3: add another item
		expected.ContainerID = "test_container_2"
		_, err = epr.SetItem(dt.zkConn, expected)
		t.Assert(err, IsNil)

		select {
		case count := <-eventC:
			t.Assert(count, Equals, 2)
		case err := <-errC:
			t.Fatalf("unexpected error running endpoint listener: %s", err)
		case <-time.After(5 * time.Second):
			t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
			// TODO: cancel listener here
		}

		// case 4: remove item
		err = epr.RemoveItem(dt.zkConn, expected.TenantID, expected.EndpointID, expected.HostID, expected.ContainerID)
		t.Assert(err, IsNil)

		select {
		case count := <-eventC:
			t.Assert(count, Equals, 1)
		case err := <-errC:
			t.Fatalf("unexpected error running endpoint listener: %s", err)
		case <-time.After(5 * time.Second):
			t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
			// TODO: cancel listener here
		}

		// case 5: remove parent
		err = dt.zkConn.Delete(path.Dir(p))
		t.Assert(err, IsNil)

		select {
		case count := <-eventC:
			t.Assert(count, Equals, 0)
		case err := <-errC:
			t.Assert(err, Equals, client.ErrNoNode)
		case <-time.After(5 * time.Second):
			t.Fatalf("Timeout from WatchTenantEndpoint: %+v", expected)
			// TODO: cancel listener here
		}
	}(epn1)
}
示例#5
0
// registerExportedEndpoints registers exported ApplicationEndpoints with zookeeper
func (c *Controller) registerExportedEndpoints() error {
	// TODO: accumulate the errors so that some endpoints get registered
	conn, err := zzk.GetLocalConnection("/")
	if err != nil {
		return err
	}

	endpointRegistry, err := registry.CreateEndpointRegistry(conn)
	if err != nil {
		glog.Errorf("Could not get EndpointRegistry. Endpoints not registered: %v", err)
		return err
	}

	var vhostRegistry *registry.VhostRegistry
	vhostRegistry, err = registry.VHostRegistry(conn)
	if err != nil {
		glog.Errorf("Could not get vhost registy. Endpoints not registered: %v", err)
		return err
	}

	c.vhostZKPaths = []string{}
	c.exportedEndpointZKPaths = []string{}

	// register exported endpoints
	for key, exportList := range c.exportedEndpoints {
		for _, export := range exportList {
			endpoint := export.endpoint
			for _, vhost := range export.vhosts {
				epName := fmt.Sprintf("%s_%v", export.endpointName, export.endpoint.InstanceID)
				//delete any existing vhost that hasn't been cleaned up
				vhostEndpoint := registry.NewVhostEndpoint(epName, endpoint)
				if paths, err := vhostRegistry.GetChildren(conn, vhost); err != nil {
					glog.V(1).Infof("error trying to clean out previous vhosts", err)
				} else {
					glog.V(1).Infof("cleaning vhost paths %v", paths)
					//clean paths
					for _, path := range paths {
						if vep, err := vhostRegistry.GetItem(conn, path); err != nil {
							glog.V(1).Infof("Could not read %s", path)
						} else {
							glog.V(4).Infof("checking instance id of %#v equal %v", vep, c.options.Service.InstanceID)
							if strconv.Itoa(vep.InstanceID) == c.options.Service.InstanceID {
								glog.V(1).Infof("Deleting stale vhost registration for %v at %v ", vhost, path)
								conn.Delete(path)
							}
						}
					}
				}

				// TODO: avoid set if item already exist with data we want
				var path string
				if path, err = vhostRegistry.SetItem(conn, vhost, vhostEndpoint); err != nil {
					glog.Errorf("could not register vhost %s for %s: %v", vhost, epName, err)
					return err
				} else {
					glog.Infof("Registered vhost %s for %s at %s", vhost, epName, path)
					c.vhostZKPaths = append(c.vhostZKPaths, path)
				}

			}
			//delete any exisiting endpoint that hasn't been cleaned up
			if paths, err := endpointRegistry.GetChildren(conn, c.tenantID, export.endpoint.Application); err != nil {
				glog.V(1).Infof("error trying to clean previous endpoints: %s", err)
			} else {
				glog.V(1).Infof("cleaning endpoint paths %v", paths)
				//clean paths
				for _, path := range paths {
					if epn, err := endpointRegistry.GetItem(conn, path); err != nil {
						glog.V(1).Infof("Could not read %s", path)
					} else {
						glog.V(4).Infof("checking instance id of %#v equal %v", epn, c.options.Service.InstanceID)
						if strconv.Itoa(epn.InstanceID) == c.options.Service.InstanceID {
							glog.V(1).Infof("Deleting stale endpoint registration for %v at %v ", export.endpointName, path)
							conn.Delete(path)
						}
					}
				}
			}
			glog.Infof("Registering exported endpoint[%s]: %+v", key, endpoint)
			path, err := endpointRegistry.SetItem(conn, registry.NewEndpointNode(c.tenantID, export.endpoint.Application, c.hostID, c.dockerID, endpoint))
			if err != nil {
				glog.Errorf("  unable to add endpoint: %+v %v", endpoint, err)
				return err
			}
			c.exportedEndpointZKPaths = append(c.exportedEndpointZKPaths, path)
			glog.V(1).Infof("  endpoint successfully added to path: %s", path)
		}
	}
	return nil
}
示例#6
0
// watchRemotePorts watches imported endpoints and updates proxies
func (c *Controller) watchRemotePorts() {
	/*
		watch each tenant endpoint
			- when endpoints are added, add the endpoint proxy if not already added
			- when endpoints are added, add watch on that endpoint for updates
			- when endpoints are deleted, tell that endpoint proxy to stop proxying - done with ephemeral znodes
			- when endpoints are deleted, may not need to deal with removing watch on that endpoint since that watch will block forever
			- deal with import regexes, i.e mysql_.*
		- may not need to initially deal with removal of tenant endpoint
	*/
	cMuxPort = uint16(c.options.Mux.Port)
	cMuxTLS = c.options.Mux.TLS

	for key, endpoint := range c.importedEndpoints {
		glog.V(2).Infof("importedEndpoints[%s]: %+v", key, endpoint)
	}

	var err error
	c.zkInfo, err = getAgentZkInfo(c.options.ServicedEndpoint)
	if err != nil {
		glog.Errorf("Invalid zk info: %v", err)
		return
	}

	zkConn, err := zzk.GetLocalConnection("/")
	if err != nil {
		glog.Errorf("watchRemotePorts - error getting zk connection: %v", err)
		return
	}

	endpointRegistry, err := registry.CreateEndpointRegistry(zkConn)
	if err != nil {
		glog.Errorf("watchRemotePorts - error getting vhost registry: %v", err)
		return
	}

	//translate closing call to endpoint cancel
	cancelEndpointWatch := make(chan bool)
	go func() {
		select {
		case errc := <-c.closing:
			glog.Infof("Closing endpoint watchers")
			select {
			case endpointsWatchCanceller <- true:
			default:
			}
			close(cancelEndpointWatch)
			errc <- nil
		}
	}()

	processTenantEndpoints := func(conn coordclient.Connection, parentPath string, tenantEndpointIDs ...string) {
		glog.V(2).Infof("processTenantEndpoints for path: %s tenantEndpointIDs: %s", parentPath, tenantEndpointIDs)

		// cancel watcher on top level /endpoints if all watchers on imported endpoints have been set up
		{
			ignorePrefix := fmt.Sprintf("%s_controlplane", c.tenantID)
			missingWatchers := false
			for id := range c.importedEndpoints {
				if strings.HasPrefix(id, ignorePrefix) {
					// ignore controlplane special imports for now - handleRemotePorts starts proxies for those right now
					// TODO: register controlplane special imports in isvcs and watch for them
					continue
				}
				if _, ok := watchers[id]; !ok {
					missingWatchers = true
				}
			}
			if !missingWatchers {
				glog.V(2).Infof("all imports are being watched - cancelling watcher on /endpoints")
				select {
				case endpointsWatchCanceller <- true:
					return
				default:
					return
				}
			}
		}

		// setup watchers for each imported tenant endpoint
		watchTenantEndpoints := func(tenantEndpointKey string) {
			glog.V(2).Infof("  watching tenantEndpointKey: %s", tenantEndpointKey)
			for {

				glog.Infof("Starting watch for tenantEndpointKey %s: %v", tenantEndpointKey, err)
				if err := endpointRegistry.WatchTenantEndpoint(zkConn, tenantEndpointKey,
					c.processTenantEndpoint, endpointWatchError, cancelEndpointWatch); err != nil {
					glog.Errorf("error watching tenantEndpointKey %s: %v", tenantEndpointKey, err)
				}
				select {
				case <-cancelEndpointWatch:
					glog.Infof("Closing watch for tenantEndpointKey %s", tenantEndpointKey)
					return
				case <-time.After(500 * time.Millisecond): //prevent tight loop
				}
			}
		}

		for _, id := range tenantEndpointIDs {
			glog.V(2).Infof("checking need to watch tenantEndpoint: %s %s", parentPath, id)

			// add watchers if they don't exist for a tenantid_application
			// and if tenant-endpoint is an import
			if _, ok := watchers[id]; !ok {
				if _, ok := c.importedEndpoints[id]; ok {
					watchers[id] = true
					go watchTenantEndpoints(id)
				} else {
					// look for imports with regexes that match each tenantEndpointID
					ep := c.getMatchingEndpoint(id)
					if ep != nil {
						watchers[id] = true
						go watchTenantEndpoints(id)
					} else {
						glog.V(2).Infof("  no need to add - not imported: %s %s for importedEndpoints: %+v", parentPath, id, c.importedEndpoints)
					}
				}
			} else {
				glog.V(2).Infof("  no need to add - existing watch tenantEndpoint: %s %s", parentPath, id)
			}

			// BEWARE: only need to deal with add, currently no need to deal with deletes
			// since tenant endpoints are currently not deleted.  only the hostid_containerid
			// entries within tenantid_application are added/deleted
		}

	}

	glog.V(2).Infof("watching endpointRegistry")
	go endpointRegistry.WatchRegistry(zkConn, endpointsWatchCanceller, processTenantEndpoints, endpointWatchError)
}