// GetHealthCheck returns the health check configuration for a service, if it exists
func (a *HostAgent) GetHealthCheck(req HealthCheckRequest, healthChecks *map[string]domain.HealthCheck) error {
glog.V(4).Infof("ControlPlaneAgent.GetHealthCheck()")
*healthChecks = make(map[string]domain.HealthCheck, 0)
controlClient, err := NewControlClient(a.master)
if err != nil {
glog.Errorf("Could not start ControlPlane client %v", err)
return err
}
defer controlClient.Close()
var svc service.Service
err = controlClient.GetService(req.ServiceID, &svc)
if err != nil {
return err
}
getSvc := func(svcID string) (service.Service, error) {
svc := service.Service{}
err := controlClient.GetService(svcID, &svc)
return svc, err
}
findChild := func(svcID, childName string) (service.Service, error) {
svc := service.Service{}
err := controlClient.FindChildService(dao.FindChildRequest{svcID, childName}, &svc)
return svc, err
}
svc.EvaluateHealthCheckTemplate(getSvc, findChild, req.InstanceID)
if svc.HealthChecks != nil {
*healthChecks = svc.HealthChecks
}
return nil
}
func (a *HostAgent) GetService(serviceID string, response *service.Service) (err error) {
*response = service.Service{}
controlClient, err := NewControlClient(a.master)
if err != nil {
glog.Errorf("Could not start ControlPlane client %v", err)
return nil
}
defer controlClient.Close()
err = controlClient.GetService(serviceID, response)
if response == nil {
*response = service.Service{}
}
if err != nil {
return err
}
getSvc := func(svcID string) (service.Service, error) {
svc := service.Service{}
err := controlClient.GetService(svcID, &svc)
return svc, err
}
findChild := func(svcID, childName string) (service.Service, error) {
svc := service.Service{}
err := controlClient.FindChildService(dao.FindChildRequest{svcID, childName}, &svc)
return svc, err
}
return response.Evaluate(getSvc, findChild, 0)
}
func (dt *DaoTest) TestDao_NewSnapshot(t *C) {
t.Skip("TODO: fix this test")
// this is technically not a unit test since it depends on the leader
// starting a watch for snapshot requests and the code here is time
// dependent waiting for that leader to start the watch
return
glog.V(0).Infof("TestDao_NewSnapshot started")
defer glog.V(0).Infof("TestDao_NewSnapshot finished")
time.Sleep(2 * time.Second) // wait for Leader to start watching for snapshot requests
service := service.Service{}
service.ID = "service-without-quiesce"
dt.Dao.RemoveService(service.ID, &unused)
// snapshot should work for services without Snapshot Pause/Resume
err := dt.Dao.AddService(service, &id)
if err != nil {
t.Fatalf("Failure creating service %+v with error: %s", service, err)
}
service.ID = "service1-quiesce"
dt.Dao.RemoveService(service.ID, &unused)
service.Snapshot.Pause = fmt.Sprintf("STATE=paused echo %s quiesce $STATE", service.ID)
service.Snapshot.Resume = fmt.Sprintf("STATE=resumed echo %s quiesce $STATE", service.ID)
err = dt.Dao.AddService(service, &id)
if err != nil {
t.Fatalf("Failure creating service %+v with error: %s", service, err)
}
service.ID = "service2-quiesce"
dt.Dao.RemoveService(service.ID, &unused)
service.Snapshot.Pause = fmt.Sprintf("STATE=paused echo %s quiesce $STATE", service.ID)
service.Snapshot.Resume = fmt.Sprintf("STATE=resumed echo %s quiesce $STATE", service.ID)
err = dt.Dao.AddService(service, &id)
if err != nil {
t.Fatalf("Failure creating service %+v with error: %s", service, err)
}
err = dt.Dao.Snapshot(service.ID, &id)
if err != nil {
t.Fatalf("Failure creating snapshot for service %+v with error: %s", service, err)
}
if id == "" {
t.Fatalf("Failure creating snapshot for service %+v - label is empty", service)
}
glog.V(0).Infof("successfully created 1st snapshot with label:%s", id)
err = dt.Dao.Snapshot(service.ID, &id)
if err != nil {
t.Fatalf("Failure creating snapshot for service %+v with error: %s", service, err)
}
if id == "" {
t.Fatalf("Failure creating snapshot for service %+v - label is empty", service)
}
glog.V(0).Infof("successfully created 2nd snapshot with label:%s", id)
time.Sleep(10 * time.Second)
}
// Use the Context field of the given template to fill in all the templates in
// the Command fields of the template's ServiceDefinitions
func injectContext(s *service.Service, svcState *servicestate.ServiceState, cp dao.ControlPlane) error {
getSvc := func(svcID string) (service.Service, error) {
svc := service.Service{}
err := cp.GetService(svcID, &svc)
return svc, err
}
findChild := func(svcID, childName string) (service.Service, error) {
svc := service.Service{}
err := cp.FindChildService(dao.FindChildRequest{svcID, childName}, &svc)
return svc, err
}
return s.Evaluate(getSvc, findChild, svcState.InstanceID)
}
func (dt *DaoTest) TestDao_UpdateService(t *C) {
dt.Dao.RemoveService("default", &unused)
svc, _ := service.NewService()
svc.ID = "default0"
svc.Name = "default0"
svc.PoolID = "default"
svc.Launch = "auto"
svc.DeploymentID = "deployment_id"
err := dt.Dao.AddService(*svc, &id)
t.Assert(err, IsNil)
svc.Name = "name"
err = dt.Dao.UpdateService(*svc, &unused)
if err != nil {
t.Errorf("Failure updating service %-v with error: %s", svc, err)
t.Fail()
}
result := service.Service{}
dt.Dao.GetService("default0", &result)
//XXX the time.Time types fail comparison despite being equal...
// as far as I can tell this is a limitation with Go
result.UpdatedAt = svc.UpdatedAt
result.CreatedAt = svc.CreatedAt
if !svc.Equals(&result) {
t.Errorf("Expected Service %+v, Actual Service %+v", result, *svc)
t.Fail()
}
svc, _ = service.NewService()
svc.ID = "default1"
svc.Name = "default1"
svc.PoolID = "default"
svc.Launch = "auto"
svc.DeploymentID = "deployment_id"
err = dt.Dao.AddService(*svc, &id)
t.Assert(err, IsNil)
svc.Name = "name"
err = dt.Dao.UpdateService(*svc, &unused)
if err == nil {
t.Errorf("Expected error updating service with same name and parent", svc)
t.Fail()
}
}
// restRemoveVirtualHost removes a vhost name from provided service and endpoint. Parameters are defined in path.
func restRemoveVirtualHost(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
serviceID, err := url.QueryUnescape(r.PathParam("serviceId"))
if err != nil {
glog.Errorf("Failed getting serviceId: %v", err)
restBadRequest(w, err)
return
}
application, err := url.QueryUnescape(r.PathParam("application"))
if err != nil {
glog.Errorf("Failed getting application: %v", err)
restBadRequest(w, err)
return
}
hostname, err := url.QueryUnescape(r.PathParam("name"))
if err != nil {
glog.Errorf("Failed getting hostname: %v", err)
restBadRequest(w, err)
return
}
var service service.Service
err = client.GetService(serviceID, &service)
if err != nil {
glog.Errorf("Unexpected error getting service (%s): %v", serviceID, err)
restServerError(w, err)
return
}
err = service.RemoveVirtualHost(application, hostname)
if err != nil {
glog.Errorf("Unexpected error removing vhost, %s, from service (%s): %v", hostname, serviceID, err)
restServerError(w, err)
return
}
var unused int
err = client.UpdateService(service, &unused)
if err != nil {
glog.Errorf("Unexpected error removing vhost, %s, from service (%s): %v", hostname, serviceID, err)
restServerError(w, err)
return
}
restSuccess(w)
}
func (f *Facade) getTenantIDAndPath(ctx datastore.Context, svc service.Service) (string, string, error) {
gs := func(id string) (service.Service, error) {
return f.getService(ctx, id)
}
tenantID, err := f.GetTenantID(ctx, svc.ID)
if err != nil {
return "", "", err
}
path, err := svc.GetPath(gs)
if err != nil {
return "", "", err
}
return tenantID, path, err
}
func (dt *DaoTest) TestDao_NewService(t *C) {
svc := service.Service{}
err := dt.Dao.AddService(svc, &id)
if err == nil {
t.Errorf("Expected failure to create service %-v", svc)
t.Fail()
}
svc.ID = "default"
svc.Name = "default"
svc.PoolID = "default"
svc.Launch = "auto"
svc.DeploymentID = "deployment_id"
err = dt.Dao.AddService(svc, &id)
if err != nil {
t.Errorf("Failure creating service %-v with error: %s", svc, err)
t.Fail()
}
err = dt.Dao.AddService(svc, &id)
if err == nil {
t.Errorf("Expected error creating redundant service %-v", svc)
t.Fail()
}
svc.ID = ""
err = dt.Dao.AddService(svc, &id)
if err == nil {
t.Errorf("Expected error creating service with same name and parent", svc)
t.Fail()
}
}
// AddService adds a service; return error if service already exists
func (f *Facade) AddService(ctx datastore.Context, svc service.Service) error {
glog.V(2).Infof("Facade.AddService: %+v", svc)
store := f.serviceStore
_, err := store.Get(ctx, svc.ID)
if err != nil && !datastore.IsErrNoSuchEntity(err) {
return err
} else if err == nil {
return fmt.Errorf("error adding service; %v already exists", svc.ID)
}
// verify the service with parent ID does not exist with the given name
if s, err := store.FindChildService(ctx, svc.DeploymentID, svc.ParentServiceID, svc.Name); err != nil {
glog.Errorf("Could not verify service path for %s: %s", svc.Name, err)
return err
} else if s != nil {
err := fmt.Errorf("service %s found at %s", svc.Name, svc.ParentServiceID)
glog.Errorf("Cannot create service %s: %s", svc.Name, err)
return err
}
// Strip the database version; we already know this is a create
svc.DatabaseVersion = 0
// Save a copy for checking configs later
svcCopy := svc
err = store.Put(ctx, &svc)
if err != nil {
glog.V(2).Infof("Facade.AddService: %+v", err)
return err
}
glog.V(2).Infof("Facade.AddService: id %+v", svc.ID)
// Compare the incoming config files to see if there are modifications from
// the original. If there are, we need to perform an update to add those
// modifications to the service.
if svcCopy.OriginalConfigs != nil && !reflect.DeepEqual(svcCopy.OriginalConfigs, svcCopy.ConfigFiles) {
// Get the current service in order to get the database version. We
// don't save this because it won't have any of the updated config
// files, among other things.
cursvc, err := store.Get(ctx, svc.ID)
if err != nil {
glog.V(2).Infof("Facade.AddService: %+v", err)
return err
}
svcCopy.DatabaseVersion = cursvc.DatabaseVersion
for key, _ := range svcCopy.OriginalConfigs {
glog.V(2).Infof("Facade.AddService: calling updateService for %s due to OriginalConfigs of %+v", svc.Name, key)
}
return f.updateService(ctx, &svcCopy)
}
glog.V(2).Infof("Facade.AddService: calling zk.updateService for %s %d ConfigFiles", svc.Name, len(svc.ConfigFiles))
return zkAPI(f).UpdateService(&svc)
}
func TestServicedCLI_CmdServiceList_one(t *testing.T) {
serviceID := "test-service-1"
expected, err := DefaultServiceAPITest.GetService(serviceID)
if err != nil {
t.Fatal(err)
}
var actual service.Service
output := pipe(InitServiceAPITest, "serviced", "service", "list", serviceID)
if err := json.Unmarshal(output, &actual); err != nil {
t.Fatalf("error unmarshaling resource: %s", err)
}
// Did you remember to update Service.Equals?
if !actual.Equals(expected) {
t.Fatalf("\ngot:\n%+v\nwant:\n%+v", actual, expected)
}
}
func (dt *DaoTest) TestDao_GetService(t *C) {
svc, _ := service.NewService()
svc.Name = "testname"
svc.PoolID = "default"
svc.Launch = "auto"
svc.DeploymentID = "deployment_id"
err := dt.Dao.AddService(*svc, &id)
t.Assert(err, IsNil)
var result service.Service
err = dt.Dao.GetService(svc.ID, &result)
t.Assert(err, IsNil)
//XXX the time.Time types fail comparison despite being equal...
// as far as I can tell this is a limitation with Go
result.UpdatedAt = svc.UpdatedAt
result.CreatedAt = svc.CreatedAt
if !svc.Equals(&result) {
t.Errorf("GetService Failed: expected=%+v, actual=%+v", svc, result)
}
}
func (f *Facade) UpdateService(ctx datastore.Context, svc service.Service) error {
glog.V(2).Infof("Facade.UpdateService: %+v", svc)
//cannot update service without validating it.
if svc.DesiredState != int(service.SVCStop) {
if err := f.validateServicesForStarting(ctx, &svc); err != nil {
glog.Warningf("Could not validate service %s (%s) for starting: %s", svc.Name, svc.ID, err)
svc.DesiredState = int(service.SVCStop)
}
for _, ep := range svc.GetServiceVHosts() {
for _, vh := range ep.VHosts {
//check that vhosts aren't already started elsewhere
if err := zkAPI(f).CheckRunningVHost(vh, svc.ID); err != nil {
return err
}
}
}
}
return f.updateService(ctx, &svc)
}
// UpdateServiceVhosts updates vhosts of a service
func UpdateServiceVhosts(conn client.Connection, svc *service.Service) error {
glog.V(2).Infof("UpdateServiceVhosts for ID:%s Name:%s", svc.ID, svc.Name)
// generate map of current vhosts
currentvhosts := map[string]string{}
if svcvhosts, err := conn.Children(zkServiceVhosts); err == client.ErrNoNode {
/*
// do not do this, otherwise, nodes aren't deleted when calling RemoveServiceVhost
if exists, err := zzk.PathExists(conn, zkServiceVhosts); err != nil {
return err
} else if !exists {
err := conn.CreateDir(zkServiceVhosts)
if err != client.ErrNodeExists && err != nil {
return err
}
}
*/
} else if err != nil {
glog.Errorf("UpdateServiceVhosts unable to retrieve vhost children at path %s %s", zkServiceVhosts, err)
return err
} else {
for _, svcvhost := range svcvhosts {
parts := strings.SplitN(svcvhost, "_", 2)
vhostname := parts[1]
currentvhosts[svcvhost] = vhostname
}
}
glog.V(2).Infof(" currentvhosts %+v", currentvhosts)
// generate map of vhosts in the service
svcvhosts := map[string]string{}
for _, ep := range svc.GetServiceVHosts() {
for _, vhostname := range ep.VHosts {
svcvhosts[fmt.Sprintf("%s_%s", svc.ID, vhostname)] = vhostname
}
}
glog.V(2).Infof(" svcvhosts %+v", svcvhosts)
// remove vhosts in current not in svc that match serviceid
for sv, vhostname := range currentvhosts {
svcID := strings.SplitN(sv, "_", 2)[0]
if svcID != svc.ID {
continue
}
if _, ok := svcvhosts[sv]; !ok {
if err := RemoveServiceVhost(conn, svc.ID, vhostname); err != nil {
return err
}
}
}
// add vhosts from svc not in current
for sv, vhostname := range svcvhosts {
if _, ok := currentvhosts[sv]; !ok {
if err := UpdateServiceVhost(conn, svc.ID, vhostname); err != nil {
return err
}
}
}
return nil
}
// restAddVirtualHost parses payload, adds the vhost to the service, then updates the service
func restAddVirtualHost(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
var request virtualHostRequest
err := r.DecodeJsonPayload(&request)
if err != nil {
restBadRequest(w, err)
return
}
var services []service.Service
var serviceRequest dao.ServiceRequest
if err := client.GetServices(serviceRequest, &services); err != nil {
glog.Errorf("Could not get services: %v", err)
restServerError(w, err)
return
}
var service *service.Service
for _, _service := range services {
if _service.ID == request.ServiceID {
service = &_service
break
}
}
if service == nil {
glog.Errorf("Could not find service: %s", services)
restServerError(w, err)
return
}
//checkout other virtual hosts for redundancy
_vhost := strings.ToLower(request.VirtualHostName)
for _, service := range services {
if service.Endpoints == nil {
continue
}
for _, endpoint := range service.Endpoints {
for _, host := range endpoint.VHosts {
if host == _vhost {
glog.Errorf("vhost %s already defined for service: %s", request.VirtualHostName, service.ID)
restServerError(w, err)
return
}
}
}
}
err = service.AddVirtualHost(request.Application, request.VirtualHostName)
if err != nil {
glog.Errorf("Unexpected error adding vhost to service (%s): %v", service.Name, err)
restServerError(w, err)
return
}
var unused int
err = client.UpdateService(*service, &unused)
if err != nil {
glog.Errorf("Unexpected error adding vhost to service (%s): %v", service.Name, err)
restServerError(w, err)
return
}
restSuccess(w)
}
// updateService internal method to use when service has been validated
func (f *Facade) updateService(ctx datastore.Context, svc *service.Service) error {
id := strings.TrimSpace(svc.ID)
if id == "" {
return errors.New("empty Service.ID not allowed")
}
svc.ID = id
//add assignment info to service so it is availble in zk
f.fillServiceAddr(ctx, svc)
svcStore := f.serviceStore
// verify the service with name and parent does not collide with another existing service
if s, err := svcStore.FindChildService(ctx, svc.DeploymentID, svc.ParentServiceID, svc.Name); err != nil {
glog.Errorf("Could not verify service path for %s: %s", svc.Name, err)
return err
} else if s != nil {
if s.ID != svc.ID {
err := fmt.Errorf("service %s found at %s", svc.Name, svc.ParentServiceID)
glog.Errorf("Cannot update service %s: %s", svc.Name, err)
return err
}
}
oldSvc, err := svcStore.Get(ctx, svc.ID)
if err != nil {
return err
}
//Deal with Service Config Files
//For now always make sure originalConfigs stay the same, essentially they are immutable
svc.OriginalConfigs = oldSvc.OriginalConfigs
//check if config files haven't changed
if !reflect.DeepEqual(oldSvc.OriginalConfigs, svc.ConfigFiles) {
//lets validate Service before doing more work....
if err := svc.ValidEntity(); err != nil {
return err
}
tenantID, servicePath, err := f.getTenantIDAndPath(ctx, *svc)
if err != nil {
return err
}
newConfs := make(map[string]*serviceconfigfile.SvcConfigFile)
//config files are different, for each one that is different validate and add to newConfs
for key, oldConf := range oldSvc.OriginalConfigs {
if conf, found := svc.ConfigFiles[key]; found {
if !reflect.DeepEqual(oldConf, conf) {
newConf, err := serviceconfigfile.New(tenantID, servicePath, conf)
if err != nil {
return err
}
newConfs[key] = newConf
}
}
}
//Get current stored conf files and replace as needed
configStore := serviceconfigfile.NewStore()
existingConfs, err := configStore.GetConfigFiles(ctx, tenantID, servicePath)
if err != nil {
return err
}
foundConfs := make(map[string]*serviceconfigfile.SvcConfigFile)
for _, svcConfig := range existingConfs {
foundConfs[svcConfig.ConfFile.Filename] = svcConfig
}
//add or replace stored service config
for _, newConf := range newConfs {
if existing, found := foundConfs[newConf.ConfFile.Filename]; found {
newConf.ID = existing.ID
//delete it from stored confs, left overs will be deleted from DB
delete(foundConfs, newConf.ConfFile.Filename)
}
configStore.Put(ctx, serviceconfigfile.Key(newConf.ID), newConf)
}
//remove leftover non-updated stored confs, conf was probably reverted to original or no longer exists
for _, confToDelete := range foundConfs {
configStore.Delete(ctx, serviceconfigfile.Key(confToDelete.ID))
}
}
svc.UpdatedAt = time.Now()
if err := svcStore.Put(ctx, svc); err != nil {
return err
}
// Remove the service from zookeeper if the pool ID has changed
if oldSvc.PoolID != svc.PoolID {
if err := zkAPI(f).RemoveService(oldSvc); err != nil {
// Synchronizer will eventually clean this service up
glog.Warningf("ZK: Could not delete service %s (%s) from pool %s: %s", svc.Name, svc.ID, oldSvc.PoolID, err)
oldSvc.DesiredState = int(service.SVCStop)
zkAPI(f).UpdateService(oldSvc)
}
}
return zkAPI(f).UpdateService(svc)
}
func restAddService(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
var svc service.Service
var serviceID string
err := r.DecodeJsonPayload(&svc)
if err != nil {
glog.V(1).Info("Could not decode service payload: ", err)
restBadRequest(w, err)
return
}
if id, err := utils.NewUUID36(); err != nil {
restBadRequest(w, err)
return
} else {
svc.ID = id
}
now := time.Now()
svc.CreatedAt = now
svc.UpdatedAt = now
//for each endpoint, evaluate its EndpointTemplates
getSvc := func(svcID string) (service.Service, error) {
svc := service.Service{}
err := client.GetService(svcID, &svc)
return svc, err
}
findChild := func(svcID, childName string) (service.Service, error) {
svc := service.Service{}
err := client.FindChildService(dao.FindChildRequest{svcID, childName}, &svc)
return svc, err
}
if err = svc.EvaluateEndpointTemplates(getSvc, findChild); err != nil {
glog.Errorf("Unable to evaluate service endpoints: %v", err)
restServerError(w, err)
return
}
tags := map[string][]string{
"controlplane_service_id": []string{svc.ID},
}
profile, err := svc.MonitoringProfile.ReBuild("1h-ago", tags)
if err != nil {
glog.Errorf("Unable to rebuild service monitoring profile: %v", err)
restServerError(w, err)
return
}
svc.MonitoringProfile = *profile
//add the service to the data store
err = client.AddService(svc, &serviceID)
if err != nil {
glog.Errorf("Unable to add service: %v", err)
restServerError(w, err)
return
}
//automatically assign virtual ips to new service
request := dao.AssignmentRequest{ServiceID: svc.ID, IPAddress: "", AutoAssignment: true}
if err := client.AssignIPs(request, nil); err != nil {
glog.Errorf("Failed to automatically assign IPs: %+v -> %v", request, err)
restServerError(w, err)
return
}
glog.V(0).Info("Added service ", serviceID)
w.WriteJson(&simpleResponse{"Added service", serviceLinks(serviceID)})
}
// sync synchronizes the number of running instances for this service
func (l *ServiceListener) sync(svc *service.Service, rss []dao.RunningService) bool {
// sort running services by instance ID, so that you stop instances by the
// lowest instance ID first and start instances with the greatest instance
// ID last.
sort.Sort(instances(rss))
// resume any paused running services
for _, state := range rss {
// resumeInstance updates the service state ONLY if it has a PAUSED DesiredState
if err := resumeInstance(l.conn, state.HostID, state.ID); err != nil {
glog.Warningf("Could not resume paused service instance %s (%s) for service %s on host %s: %s", state.ID, state.Name, state.ServiceID, state.HostID, err)
}
}
// if the service has a change option for restart all on changed, stop all
// instances and wait for the nodes to stop. Once all service instances
// have been stopped (deleted), then go ahead and start the instances back
// up.
if count := len(rss); count > 0 && count != svc.Instances && utils.StringInSlice("restartAllOnInstanceChanged", svc.ChangeOptions) {
svc.Instances = 0 // NOTE: this will not update the node in zk or elastic
}
// netInstances is the difference between the number of instances that
// should be running, as described by the service from the number of
// instances that are currently running
netInstances := svc.Instances - len(rss)
if netInstances > 0 {
// If the service lock is enabled, do not try to start any service instances
// This will prevent the retry restart from activating
if locked, err := IsServiceLocked(l.conn); err != nil {
glog.Errorf("Could not check service lock: %s", err)
return true
} else if locked {
glog.Warningf("Could not start %d instances; service %s (%s) is locked", netInstances, svc.Name, svc.ID)
return true
}
// the number of running instances is *less* than the number of
// instances that need to be running, so schedule instances to start
glog.V(2).Infof("Starting %d instances of service %s (%s)", netInstances, svc.Name, svc.ID)
var (
last = 0
instanceIDs = make([]int, netInstances)
)
// Find which instances IDs are being unused and add those instances
// first. All SERVICES must have an instance ID of 0, if instance ID
// zero dies for whatever reason, then the service must schedule
// another 0-id instance to take its place.
j := 0
for i := range instanceIDs {
for j < len(rss) && last == rss[j].InstanceID {
// if instance ID exists, then keep searching the list for
// the next unique instance ID
last += 1
j += 1
}
instanceIDs[i] = last
last += 1
}
return netInstances == l.start(svc, instanceIDs)
} else if netInstances = -netInstances; netInstances > 0 {
// the number of running instances is *greater* than the number of
// instances that need to be running, so schedule instances to stop of
// the highest instance IDs.
glog.V(2).Infof("Stopping %d of %d instances of service %s (%s)", netInstances, len(rss), svc.Name, svc.ID)
l.stop(rss[svc.Instances:])
}
return true
}