Esempio n. 1
0
// start schedules the given service instances with the proviced instance ID.
func (l *ServiceListener) start(svc *service.Service, instanceIDs []int) int {
	var i, id int

	for i, id = range instanceIDs {
		if success := func(instanceID int) bool {
			glog.V(2).Infof("Waiting to acquire scheduler lock for service %s (%s)", svc.Name, svc.ID)
			// only one service instance can be scheduled at a time
			l.Lock()
			defer l.Unlock()

			// If the service lock is enabled, do not try to start the service instance
			glog.V(2).Infof("Scheduler lock acquired for service %s (%s); checking service lock", svc.Name, svc.ID)
			if locked, err := IsServiceLocked(l.conn); err != nil {
				glog.Errorf("Could not check service lock: %s", err)
				return false
			} else if locked {
				glog.Warningf("Could not start instance %d; service %s (%s) is locked", instanceID, svc.Name, svc.ID)
				return false
			}

			glog.V(2).Infof("Service is not locked, selecting a host for service %s (%s) #%d", svc.Name, svc.ID, id)

			host, err := l.handler.SelectHost(svc)
			if err != nil {
				glog.Warningf("Could not assign a host to service %s (%s): %s", svc.Name, svc.ID, err)
				return false
			}

			glog.V(2).Infof("Host %s found, building service instance %d for %s (%s)", host.ID, id, svc.Name, svc.ID)

			state, err := servicestate.BuildFromService(svc, host.ID)
			if err != nil {
				glog.Warningf("Error creating service state for service %s (%s): %s", svc.Name, svc.ID, err)
				return false
			}

			state.HostIP = host.IPAddr
			state.InstanceID = instanceID
			if err := addInstance(l.conn, *state); err != nil {
				glog.Warningf("Could not add service instance %s for service %s (%s): %s", state.ID, svc.Name, svc.ID, err)
				return false
			}
			glog.V(2).Infof("Starting service instance %s for service %s (%s) on host %s", state.ID, svc.Name, svc.ID, host.ID)
			return true
		}(id); !success {
			// 'i' is the index of the unsuccessful instance id which should portray
			// the number of successful instances.  If you have 2 successful instances
			// started, then i = 2 because it attempted to create the third index and
			// failed
			glog.Warningf("Started %d of %d service instances for %s (%s)", i, len(instanceIDs), svc.Name, svc.ID)
			return i
		}
	}
	// add 1 because the index of the last instance 'i' would be len(instanceIDs) - 1
	return i + 1
}
Esempio n. 2
0
func (t *ZZKTest) TestGetServiceStatus(c *C) {
	conn, err := zzk.GetLocalConnection("/")
	c.Assert(err, IsNil)

	// Add a service
	svc := service.Service{ID: "test-service-1", Instances: 3}
	err = UpdateService(conn, &svc)
	c.Assert(err, IsNil)

	// Add a host
	register := func(hostID string) string {
		c.Logf("Registering host %s", hostID)
		host := host.Host{ID: hostID}
		err := AddHost(conn, &host)
		c.Assert(err, IsNil)
		p, err := conn.CreateEphemeral(hostregpath(hostID), &HostNode{Host: &host})
		c.Assert(err, IsNil)
		return path.Base(p)
	}
	register("test-host-1")

	// Case 1: Zero service states
	statusmap, err := GetServiceStatus(conn, svc.ID)
	c.Assert(err, IsNil)
	c.Assert(statusmap, NotNil)
	c.Assert(statusmap, DeepEquals, map[string]dao.ServiceStatus{})

	// Add states
	addstates := func(hostID string, svc *service.Service, count int) []string {
		c.Logf("Adding %d service states for service %s on host %s", count, svc.ID, hostID)
		stateIDs := make([]string, count)
		for i := 0; i < count; i++ {
			state, err := servicestate.BuildFromService(svc, hostID)
			c.Assert(err, IsNil)
			err = addInstance(conn, *state)
			c.Assert(err, IsNil)
			_, err = LoadRunningService(conn, state.ServiceID, state.ID)
			c.Assert(err, IsNil)
			stateIDs[i] = state.ID
		}
		return stateIDs
	}
	stateIDs := addstates("test-host-1", &svc, 3)

	states := make(map[string]servicestate.ServiceState)
	updatestate := func(stateID, serviceID string, update func(state *servicestate.ServiceState)) servicestate.ServiceState {
		var node ServiceStateNode
		err = conn.Get(servicepath(serviceID, stateID), &node)
		c.Assert(err, IsNil)
		update(node.ServiceState)
		err = conn.Set(servicepath(serviceID, stateID), &node)
		c.Assert(err, IsNil)
		return *node.ServiceState
	}

	// State 0 started
	states[stateIDs[0]] = updatestate(stateIDs[0], svc.ID, func(s *servicestate.ServiceState) {
		s.Started = time.Now()
	})

	// State 1 paused
	states[stateIDs[1]] = updatestate(stateIDs[1], svc.ID, func(s *servicestate.ServiceState) {
		s.Started = time.Now()
		s.Paused = true
	})

	// State 2 stopped (no update)
	states[stateIDs[2]] = updatestate(stateIDs[2], svc.ID, func(s *servicestate.ServiceState) {})

	c.Log("Desired state is RUN")
	statusmap, err = GetServiceStatus(conn, svc.ID)
	c.Assert(err, IsNil)
	c.Assert(statusmap, DeepEquals, map[string]dao.ServiceStatus{
		stateIDs[0]: dao.ServiceStatus{states[stateIDs[0]], dao.Running},
		stateIDs[1]: dao.ServiceStatus{states[stateIDs[1]], dao.Resuming},
		stateIDs[2]: dao.ServiceStatus{states[stateIDs[2]], dao.Starting},
	})

	c.Log("Desired state is PAUSE")
	for _, state := range stateIDs {
		err := pauseInstance(conn, "test-host-1", state)
		c.Assert(err, IsNil)
	}
	statusmap, err = GetServiceStatus(conn, svc.ID)
	c.Assert(err, IsNil)
	c.Assert(statusmap, DeepEquals, map[string]dao.ServiceStatus{
		stateIDs[0]: dao.ServiceStatus{states[stateIDs[0]], dao.Pausing},
		stateIDs[1]: dao.ServiceStatus{states[stateIDs[1]], dao.Paused},
		stateIDs[2]: dao.ServiceStatus{states[stateIDs[2]], dao.Stopped},
	})

	c.Log("Desired state is STOP")
	for _, state := range stateIDs {
		err := StopServiceInstance(conn, "test-host-1", state)
		c.Assert(err, IsNil)
	}
	statusmap, err = GetServiceStatus(conn, svc.ID)
	c.Assert(err, IsNil)
	c.Assert(statusmap, DeepEquals, map[string]dao.ServiceStatus{
		stateIDs[0]: dao.ServiceStatus{states[stateIDs[0]], dao.Stopping},
		stateIDs[1]: dao.ServiceStatus{states[stateIDs[1]], dao.Stopping},
		stateIDs[2]: dao.ServiceStatus{states[stateIDs[2]], dao.Stopped},
	})
}
Esempio n. 3
0
func (t *ZZKTest) TestHostStateListener_pauseANDresume(c *C) {
	conn, err := zzk.GetLocalConnection("/base_pauseANDresume")
	c.Assert(err, IsNil)

	handler := new(TestHostStateHandler).init()
	listener := NewHostStateListener(handler, "test-host-1")
	listener.SetConnection(conn)

	// Add a service
	svc := service.Service{ID: "test-service-1", Instances: 1}
	err = UpdateService(conn, &svc)
	c.Assert(err, IsNil)

	// Add a host
	register := func(hostID string) string {
		c.Logf("Registering host %s", hostID)
		host := host.Host{ID: hostID}
		err := AddHost(conn, &host)
		c.Assert(err, IsNil)
		p, err := conn.CreateEphemeral(hostregpath(hostID), &HostNode{Host: &host})
		c.Assert(err, IsNil)
		return path.Base(p)
	}
	register("test-host-1")

	// Add states
	addstates := func(hostID string, svc *service.Service, count int) []string {
		c.Logf("Adding %d service states for service %s on host %s", count, svc.ID, hostID)
		stateIDs := make([]string, count)
		for i := 0; i < count; i++ {
			state, err := servicestate.BuildFromService(svc, hostID)
			c.Assert(err, IsNil)
			err = addInstance(conn, *state)
			c.Assert(err, IsNil)
			_, err = LoadRunningService(conn, state.ServiceID, state.ID)
			c.Assert(err, IsNil)
			stateIDs[i] = state.ID
		}
		return stateIDs
	}
	stateIDs := addstates("test-host-1", &svc, 1)
	stateID := stateIDs[0]

	var node ServiceStateNode
	err = conn.Get(servicepath(svc.ID, stateID), &node)
	c.Assert(err, IsNil)
	err = handler.StartService(&svc, node.ServiceState, func(_ string) {})
	c.Assert(err, IsNil)
	err = conn.Set(servicepath(node.ServiceID, node.ID), &node)
	c.Assert(err, IsNil)
	err = listener.pauseInstance(&svc, node.ServiceState)
	c.Assert(err, IsNil)
	err = conn.Get(servicepath(node.ServiceID, node.ID), &node)
	c.Assert(err, IsNil)
	c.Assert(node.IsPaused(), Equals, true)
	err = listener.resumeInstance(&svc, node.ServiceState)
	c.Assert(err, IsNil)
	err = conn.Get(servicepath(node.ServiceID, node.ID), &node)
	c.Assert(err, IsNil)
	c.Assert(node.IsPaused(), Equals, false)
}
Esempio n. 4
0
func (t *ZZKTest) TestHostStateListener_Spawn_StartAndStop(c *C) {
	conn, err := zzk.GetLocalConnection("/base")
	c.Assert(err, IsNil)

	shutdown := make(chan interface{})
	defer close(shutdown)
	errC := make(chan error, 1)

	handler := new(TestHostStateHandler).init()
	listener := NewHostStateListener(handler, "test-host-1")
	listener.SetConnection(conn)

	// Add a service
	svc := service.Service{ID: "test-service-1", Instances: 1}
	err = UpdateService(conn, &svc)
	c.Assert(err, IsNil)

	// Add a host
	register := func(hostID string) string {
		c.Logf("Registering host %s", hostID)
		host := host.Host{ID: hostID}
		err := AddHost(conn, &host)
		c.Assert(err, IsNil)
		p, err := conn.CreateEphemeral(hostregpath(hostID), &HostNode{Host: &host})
		c.Assert(err, IsNil)
		return path.Base(p)
	}
	register("test-host-1")

	// Add states
	addstates := func(hostID string, svc *service.Service, count int) []string {
		c.Logf("Adding %d service states for service %s on host %s", count, svc.ID, hostID)
		stateIDs := make([]string, count)
		for i := 0; i < count; i++ {
			state, err := servicestate.BuildFromService(svc, hostID)
			c.Assert(err, IsNil)
			err = addInstance(conn, *state)
			c.Assert(err, IsNil)
			_, err = LoadRunningService(conn, state.ServiceID, state.ID)
			c.Assert(err, IsNil)
			stateIDs[i] = state.ID
		}
		return stateIDs
	}
	stateIDs := addstates("test-host-1", &svc, 1)
	stateID := stateIDs[0]

	wait := func(serviceID string, dState service.DesiredState) {
		c.Logf("Waiting for service instances on 'test-host-1' to %s", dState)
		go func() {
			errC <- WaitService(shutdown, conn, serviceID, dState)
		}()

		// Wait on services or fail trying
		select {
		case err := <-errC:
			c.Assert(err, IsNil)
		case <-time.After(zzk.ZKTestTimeout):
			c.Fatalf("timeout waiting for instances to %s", dState)
		}
	}

	var node1, node2 ServiceStateNode
	go func() {
		listener.Spawn(shutdown, stateID)
	}()

	c.Logf("Checking instance start")
	wait(svc.ID, service.SVCRun)
	err = conn.Get(servicepath(svc.ID, stateID), &node1)
	c.Assert(err, IsNil)

	c.Logf("Stopping service instance")
	err = handler.StopService(node1.ServiceState)
	wait(svc.ID, service.SVCRun)
	err = conn.Get(servicepath(svc.ID, stateID), &node2)
	c.Assert(err, IsNil)
	c.Assert(node2.Started.After(node1.Started), Equals, true)

	c.Logf("Pausing service instance")
	err = pauseInstance(conn, "test-host-1", stateID)
	wait(svc.ID, service.SVCPause)

	c.Logf("Resuming service instance")
	err = resumeInstance(conn, "test-host-1", stateID)
	c.Assert(err, IsNil)
	wait(svc.ID, service.SVCRun)
	// Verify the instance wasn't restarted
	err = conn.Get(servicepath(svc.ID, stateID), &node1)
	c.Assert(err, IsNil)
	c.Assert(node1.Started.Unix(), Equals, node2.Started.Unix())

	c.Logf("Stopping service instance")
	err = StopServiceInstance(conn, "test-host-1", stateID)
	wait(svc.ID, service.SVCStop)
}
Esempio n. 5
0
func (t *ZZKTest) TestHostStateListener_Spawn_Shutdown(c *C) {
	conn, err := zzk.GetLocalConnection("/base")
	c.Assert(err, IsNil)

	shutdown := make(chan interface{})

	handler := new(TestHostStateHandler).init()
	listener := NewHostStateListener(handler, "test-host-1")
	listener.SetConnection(conn)

	// Add a service
	svc := service.Service{ID: "test-service-1", Instances: 1}
	err = UpdateService(conn, &svc)
	c.Assert(err, IsNil)

	// Add a host
	register := func(hostID string) string {
		c.Logf("Registering host %s", hostID)
		host := host.Host{ID: hostID}
		err := AddHost(conn, &host)
		c.Assert(err, IsNil)
		p, err := conn.CreateEphemeral(hostregpath(hostID), &HostNode{Host: &host})
		c.Assert(err, IsNil)
		return path.Base(p)
	}
	register("test-host-1")

	// Add states
	addstates := func(hostID string, svc *service.Service, count int) []string {
		c.Logf("Adding %d service states for service %s on host %s", count, svc.ID, hostID)
		stateIDs := make([]string, count)
		for i := 0; i < count; i++ {
			state, err := servicestate.BuildFromService(svc, hostID)
			c.Assert(err, IsNil)
			err = addInstance(conn, *state)
			c.Assert(err, IsNil)
			_, err = LoadRunningService(conn, state.ServiceID, state.ID)
			c.Assert(err, IsNil)
			stateIDs[i] = state.ID
		}
		return stateIDs
	}
	stateIDs := addstates("test-host-1", &svc, 1)
	stateID := stateIDs[0]

	done := make(chan struct{})
	go func() {
		defer close(done)
		listener.Spawn(shutdown, stateID)
	}()

	time.Sleep(zzk.ZKTestTimeout)
	close(shutdown)

	select {
	case <-done:
	case <-time.After(zzk.ZKTestTimeout):
		c.Fatalf("timeout waiting for listener to shutdown")
	}

	exists, err := conn.Exists(servicepath(svc.ID, stateID))
	c.Assert(err, IsNil)
	c.Assert(exists, Equals, false)
	exists, err = conn.Exists(hostpath("test-host-1", stateID))
	c.Assert(err, IsNil)
	c.Assert(exists, Equals, false)
}
Esempio n. 6
0
func (t *ZZKTest) TestHostStateListener_Listen(c *C) {
	conn, err := zzk.GetLocalConnection("/base")
	c.Assert(err, IsNil)

	shutdown := make(chan interface{})
	defer close(shutdown)
	errC := make(chan error, 1)

	handler := new(TestHostStateHandler).init()
	listener := NewHostStateListener(handler, "test-host-1")
	go zzk.Listen(shutdown, errC, conn, listener)

	// Add a service
	svc := service.Service{ID: "test-service-1", Instances: 3}
	err = UpdateService(conn, &svc)
	c.Assert(err, IsNil)

	// Add host
	err = AddHost(conn, &host.Host{ID: "test-host-1"})
	c.Assert(err, IsNil)

	// Verify that the host is registered
	c.Logf("Waiting for 'test-host-1' to be registered")
	select {
	case err := <-errC:
		c.Assert(err, IsNil)
		c.Assert(listener.registry, Not(Equals), "")

		exists, err := conn.Exists(listener.registry)
		c.Assert(err, IsNil)
		c.Assert(exists, Equals, true)
	case <-time.After(zzk.ZKTestTimeout):
		// NOTE: this timeout may be adjusted to satisfy race conditions
		c.Fatalf("timeout waiting for host to be ready")
	}

	// Add states
	addstates := func(hostID string, svc *service.Service, count int) []string {
		c.Logf("Adding %d service states for service %s on host %s", count, svc.ID, hostID)
		stateIDs := make([]string, count)
		for i := 0; i < count; i++ {
			state, err := servicestate.BuildFromService(svc, hostID)
			c.Assert(err, IsNil)
			c.Assert(state.IsRunning(), Equals, false)
			err = addInstance(conn, *state)
			c.Assert(err, IsNil)
			_, err = LoadRunningService(conn, state.ServiceID, state.ID)
			c.Assert(err, IsNil)
			stateIDs[i] = state.ID
		}
		return stateIDs
	}
	stateIDs := addstates("test-host-1", &svc, 3)

	wait := func(serviceID string, dState service.DesiredState) {
		errC := make(chan error)
		c.Logf("Waiting for service instances on 'test-host-1' to %s", dState)
		go func() {
			errC <- WaitService(shutdown, conn, serviceID, dState)
		}()

		// Wait on services or fail trying
		select {
		case err := <-errC:
			c.Assert(err, IsNil)
		case <-time.After(zzk.ZKTestTimeout):
			c.Fatalf("timeout waiting for instances to %s", dState)
		}
	}
	wait(svc.ID, service.SVCRun)

	// Pause states
	for _, stateID := range stateIDs {
		err = pauseInstance(conn, "test-host-1", stateID)
		c.Assert(err, IsNil)
	}
	wait(svc.ID, service.SVCPause)

	// Resume states
	for _, stateID := range stateIDs {
		err = resumeInstance(conn, "test-host-1", stateID)
		c.Assert(err, IsNil)
	}
	wait(svc.ID, service.SVCRun)

	// Stop states
	for _, stateID := range stateIDs {
		err = StopServiceInstance(conn, "test-host-1", stateID)
		c.Assert(err, IsNil)
	}
	wait(svc.ID, service.SVCStop)
}
Esempio n. 7
0
func TestNewRunningService(t *testing.T) {
	sd := servicedefinition.ServiceDefinition{
		MonitoringProfile: domain.MonitorProfile{
			MetricConfigs: []domain.MetricConfig{
				domain.MetricConfig{
					ID:          "jvm.memory",
					Name:        "JVM Memory",
					Description: "JVM heap vs. non-heap memory usage",
					Metrics: []domain.Metric{
						domain.Metric{ID: "jvm.memory.heap", Name: "JVM Heap Usage"},
						domain.Metric{ID: "jvm.memory.non_heap", Name: "JVM Non-Heap Usage"},
					},
				},
			},
		},
	}
	svc, err := service.BuildService(sd, "", "", 0, "")
	if err != nil {
		t.Errorf("BuildService Failed w/err=%s", err)
	}
	dataHeapRequest := fmt.Sprintf("{\"metric\":\"jvm.memory.heap\",\"tags\":{\"controlplane_service_id\":[\"%s\"]}}", svc.ID)
	dataNonHeapRequest := fmt.Sprintf("{\"metric\":\"jvm.memory.non_heap\",\"tags\":{\"controlplane_service_id\":[\"%s\"]}}", svc.ID)
	data := fmt.Sprintf("{\"metrics\":[%s,%s],\"start\":\"1h-ago\"}", dataHeapRequest, dataNonHeapRequest)
	svc.MonitoringProfile = domain.MonitorProfile{
		MetricConfigs: []domain.MetricConfig{
			domain.MetricConfig{
				ID:          "jvm.memory",
				Name:        "JVM Memory",
				Description: "JVM heap vs. non-heap memory usage",
				Query: domain.QueryConfig{
					RequestURI: "/metrics/api/performance/query",
					Method:     "POST",
					Headers: map[string][]string{
						"Content-Type": []string{"application/json"},
					},
					Data: data,
				},
				Metrics: []domain.Metric{
					domain.Metric{ID: "jvm.memory.heap", Name: "JVM Heap Usage"},
				},
			},
		},
	}

	svcstate, err := servicestate.BuildFromService(svc, "fakehostid")
	if err != nil {
		t.Error("%v", err)
	}

	rs, err := NewRunningService(svc, svcstate)
	if err != nil {
		t.Error("%v", err)
	}

	var query interface{}
	json.Unmarshal([]byte(rs.MonitoringProfile.MetricConfigs[0].Query.Data), &query)

	metrics := query.(map[string]interface{})["metrics"].([]interface{})[0].(map[string]interface{})

	tags := metrics["tags"].(map[string]interface{})

	controlplaneInstanceID := tags["controlplane_instance_id"].([]interface{})[0]
	if controlplaneInstanceID != "0" {
		t.Errorf("Expected %+v, got %+v", "0", controlplaneInstanceID)
	}

	controlplaneServiceID := tags["controlplane_service_id"].([]interface{})[0]
	if controlplaneServiceID != svc.ID {
		t.Errorf("Expected %+v, got %+v", svc.ID, controlplaneServiceID)
	}
}
Esempio n. 8
0
func (t *ZZKTest) TestHostRegistryListener_unregister(c *C) {
	conn, err := zzk.GetLocalConnection("/base")
	c.Assert(err, IsNil)

	// Initialize the host registry
	err = InitHostRegistry(conn)
	c.Assert(err, IsNil)

	listener := NewHostRegistryListener()
	listener.SetConnection(conn)

	// Add a service
	svc := service.Service{ID: "test-service-1"}
	err = UpdateService(conn, &svc)
	c.Assert(err, IsNil)

	// Add hosts
	register := func(hostID string) string {
		c.Logf("Registering host %s", hostID)
		host := host.Host{ID: hostID}

		err := AddHost(conn, &host)
		c.Assert(err, IsNil)

		p, err := conn.CreateEphemeral(hostregpath(hostID), &HostNode{Host: &host})
		c.Assert(err, IsNil)

		return path.Base(p)
	}
	register("test-host-1")
	register("test-host-2")

	// Add states
	states := make(map[string][]string)
	addstates := func(hostID string, svc *service.Service, count int) []string {
		c.Logf("Adding %d service states for service %s on host %s", count, svc.ID, hostID)
		stateIDs := make([]string, count)

		for i := 0; i < count; i++ {
			state, err := servicestate.BuildFromService(svc, hostID)
			c.Assert(err, IsNil)

			err = addInstance(conn, *state)
			c.Assert(err, IsNil)

			_, err = LoadRunningService(conn, state.ServiceID, state.ID)
			c.Assert(err, IsNil)
			stateIDs[i] = state.ID
		}

		return stateIDs
	}
	states["test-host-1"] = addstates("test-host-1", &svc, 2)
	states["test-host-2"] = addstates("test-host-2", &svc, 2)

	// unregister the host instances
	c.Logf("Unregistering service instances for 'test-host-1'")
	listener.unregister("test-host-1")

	// verify states removed
	for _, state := range states["test-host-1"] {
		exists, err := conn.Exists(hostpath("test-host-1", state))
		c.Assert(err, IsNil)
		c.Assert(exists, Equals, false)
		exists, err = conn.Exists(servicepath(svc.ID, state))
		c.Assert(err, IsNil)
		c.Assert(exists, Equals, false)
	}

	// verify states preserved
	for _, state := range states["test-host-2"] {
		exists, err := conn.Exists(hostpath("test-host-2", state))
		c.Assert(err, IsNil)
		c.Assert(exists, Equals, true)
		exists, err = conn.Exists(servicepath(svc.ID, state))
		c.Assert(err, IsNil)
		c.Assert(exists, Equals, true)
	}
}
Esempio n. 9
0
func (t *ZZKTest) TestHostRegistryListener_Listen(c *C) {
	conn, err := zzk.GetLocalConnection("/base")
	c.Assert(err, IsNil)

	// Initialize the host registry
	err = InitHostRegistry(conn)
	c.Assert(err, IsNil)

	shutdown := make(chan interface{})
	defer close(shutdown)

	listener := NewHostRegistryListener()
	go zzk.Listen(shutdown, make(chan error, 1), conn, listener)

	// Add a service
	svc := service.Service{ID: "test-service-1"}
	err = UpdateService(conn, &svc)
	c.Assert(err, IsNil)

	// Add hosts
	hosts := make(map[string]string)
	register := func(hostID string) string {
		c.Logf("Registering host %s", hostID)
		host := host.Host{ID: hostID}

		err := AddHost(conn, &host)
		c.Assert(err, IsNil)

		p, err := conn.CreateEphemeral(hostregpath(hostID), &HostNode{Host: &host})
		c.Assert(err, IsNil)

		return path.Base(p)
	}
	hosts["test-host-1"] = register("test-host-1")
	hosts["test-host-2"] = register("test-host-2")

	// Add states
	addstates := func(hostID string, svc *service.Service, count int) []string {
		c.Logf("Adding %d service states for service %s on host %s", count, svc.ID, hostID)
		stateIDs := make([]string, count)

		for i := 0; i < count; i++ {
			state, err := servicestate.BuildFromService(svc, hostID)
			c.Assert(err, IsNil)

			err = addInstance(conn, *state)
			c.Assert(err, IsNil)

			_, err = LoadRunningService(conn, state.ServiceID, state.ID)
			c.Assert(err, IsNil)
			stateIDs[i] = state.ID
		}

		return stateIDs
	}
	addstates("test-host-1", &svc, 2)
	addstates("test-host-2", &svc, 2)

	// unregister a host and verify the states have been removed
	unregister := func(hostID, ehostID string) {
		var wg sync.WaitGroup

		hsids, err := conn.Children(hostpath(hostID))
		c.Assert(err, IsNil)

		// Monitor the states per service
		for _, hsid := range hsids {
			var hs HostState
			err = conn.Get(hostpath(hostID, hsid), &hs)
			c.Assert(err, IsNil)

			wg.Add(1)
			go func(hsid, serviceID string) {
				defer wg.Done()
				for {
					event, err := conn.GetW(servicepath(serviceID, hsid), &HostNode{})
					c.Assert(err, IsNil)
					if e := <-event; e.Type == client.EventNodeDeleted {
						return
					}
				}
			}(hs.ServiceStateID, hs.ServiceID)
		}

		// Monitor the host state
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				hsids, event, err := conn.ChildrenW(hostpath(hostID))
				c.Assert(err, IsNil)
				if len(hsids) == 0 {
					return
				}
				<-event
			}
		}()

		c.Logf("Unregistering host %s", hostID)
		err = conn.Delete(hostregpath(ehostID))
		c.Assert(err, IsNil)
		wg.Wait()
	}

	done := make(chan struct{})
	go func() {
		done <- struct{}{}
		unregister("test-host-1", hosts["test-host-1"])
	}()

	select {
	case <-done:
	case <-time.After(1 * time.Minute):
		c.Errorf("timeout")
	}
}
Esempio n. 10
0
// getEndpoints builds exportedEndpoints and importedEndpoints
func (c *Controller) getEndpoints(service *service.Service) error {
	var err error
	c.zkInfo, err = getAgentZkInfo(c.options.ServicedEndpoint)
	if err != nil {
		glog.Errorf("Invalid zk info: %v", err)
		return err //ErrInvalidZkInfo
	}
	glog.Infof(" c.zkInfo: %+v", c.zkInfo)

	// endpoints are created at the root level (not pool aware)
	rootBasePath := ""
	zClient, err := coordclient.New("zookeeper", c.zkInfo.ZkDSN, rootBasePath, nil)
	if err != nil {
		glog.Errorf("failed create a new coordclient: %v", err)
		return err
	}

	zzk.InitializeLocalClient(zClient)

	// get zookeeper connection
	conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(service.PoolID))
	if err != nil {
		return fmt.Errorf("getEndpoints zzk.GetLocalConnection failed: %v", err)
	}

	if os.Getenv("SERVICED_IS_SERVICE_SHELL") == "true" {
		// this is not a running service, i.e. serviced shell/run
		if hostname, err := os.Hostname(); err != nil {
			glog.Errorf("could not get hostname: %s", err)
			return fmt.Errorf("getEndpoints failed could not get hostname: %v", err)
		} else {
			c.dockerID = hostname
		}

		// TODO: deal with exports in the future when there is a use case for it

		sstate, err := servicestate.BuildFromService(service, c.hostID)
		if err != nil {
			return fmt.Errorf("Unable to create temporary service state")
		}
		// initialize importedEndpoints
		c.importedEndpoints, err = buildImportedEndpoints(conn, c.tenantID, sstate)
		if err != nil {
			glog.Errorf("Invalid ImportedEndpoints")
			return ErrInvalidImportedEndpoints
		}
	} else {
		// get service state
		glog.Infof("getting service state: %s %v", c.options.Service.ID, c.options.Service.InstanceID)
		sstate, err := getServiceState(conn, c.options.Service.ID, c.options.Service.InstanceID)
		if err != nil {
			return fmt.Errorf("getEndpoints getServiceState failed: %v", err)
		}
		c.dockerID = sstate.DockerID

		// keep a copy of the service EndPoint exports
		c.exportedEndpoints, err = buildExportedEndpoints(conn, c.tenantID, sstate)
		if err != nil {
			glog.Errorf("Invalid ExportedEndpoints")
			return ErrInvalidExportedEndpoints
		}

		// initialize importedEndpoints
		c.importedEndpoints, err = buildImportedEndpoints(conn, c.tenantID, sstate)
		if err != nil {
			glog.Errorf("Invalid ImportedEndpoints")
			return ErrInvalidImportedEndpoints
		}
	}

	return nil
}