func restDeployAppTemplate(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
var payload dao.ServiceTemplateDeploymentRequest
err := r.DecodeJsonPayload(&payload)
if err != nil {
glog.V(1).Info("Could not decode deployment payload: ", err)
restBadRequest(w, err)
return
}
var tenantID string
err = client.DeployTemplate(payload, &tenantID)
if err != nil {
glog.Error("Could not deploy template: ", err)
restServerError(w, err)
return
}
glog.V(0).Info("Deployed template ", payload)
assignmentRequest := dao.AssignmentRequest{tenantID, "", true}
if err := client.AssignIPs(assignmentRequest, nil); err != nil {
glog.Error("Could not automatically assign IPs: %v", err)
return
}
glog.Infof("Automatically assigned IP addresses to service: %v", tenantID)
// end of automatic IP assignment
w.WriteJson(&simpleResponse{tenantID, servicesLinks()})
}
func (this *ControlPlaneDao) GetRunningServices(request dao.EntityRequest, allRunningServices *[]dao.RunningService) error {
// we initialize the data container to something here in case it has not been initialized yet
*allRunningServices = make([]dao.RunningService, 0)
allPools, err := this.facade.GetResourcePools(datastore.Get())
if err != nil {
glog.Error("runningservice.go failed to get resource pool")
return err
} else if allPools == nil || len(allPools) == 0 {
return fmt.Errorf("no resource pools found")
}
for _, aPool := range allPools {
poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(aPool.ID))
if err != nil {
glog.Error("runningservice.go Failed to get connection based on pool: %v", aPool.ID)
return err
}
singlePoolRunningServices := []dao.RunningService{}
singlePoolRunningServices, err = zkservice.LoadRunningServices(poolBasedConn)
if err != nil {
glog.Errorf("Failed GetAllRunningServices: %v", err)
return err
}
for _, rs := range singlePoolRunningServices {
*allRunningServices = append(*allRunningServices, rs)
}
}
return nil
}
// listenAndMux listens for incoming connections and attempts to multiplex them
// to the local service that they request via a Zen-Service header in their
// initial message.
func (mux *TCPMux) ListenAndMux() {
var l net.Listener
var err error
if mux.UseTLS == false {
l, err = net.Listen("tcp4", fmt.Sprintf(":%d", mux.Port))
} else {
cert, cerr := tls.X509KeyPair([]byte(proxyCertPEM), []byte(proxyKeyPEM))
if cerr != nil {
glog.Error("ListenAndMux Error (tls.X509KeyPair): ", cerr)
return
}
tlsConfig := tls.Config{Certificates: []tls.Certificate{cert}}
l, err = tls.Listen("tcp4", fmt.Sprintf(":%d", mux.Port), &tlsConfig)
}
if err != nil {
glog.Error("ListenAndMux Error (net.Listen): ", err)
return
}
defer l.Close()
for {
conn, err := l.Accept()
if err != nil {
glog.Error("ListenAndMux Error (net.Accept): ", err)
return
}
go mux.MuxConnection(conn)
}
}
func (f *Facade) GetTaggedServices(ctx datastore.Context, request dao.EntityRequest) ([]service.Service, error) {
glog.V(3).Infof("Facade.GetTaggedServices")
store := f.serviceStore
switch v := request.(type) {
case []string:
results, err := store.GetTaggedServices(ctx, v...)
if err != nil {
glog.Error("Facade.GetTaggedServices: err=", err)
return nil, err
}
if err = f.fillOutServices(ctx, results); err != nil {
return nil, err
}
glog.V(2).Infof("Facade.GetTaggedServices: services=%v", results)
return results, nil
case dao.ServiceRequest:
glog.V(3).Infof("request: %+v", request)
// Get the tagged services
services, err := store.GetTaggedServices(ctx, request.(dao.ServiceRequest).Tags...)
if err != nil {
glog.Error("Facade.GetTaggedServices: err=", err)
return nil, err
}
if err = f.fillOutServices(ctx, services); err != nil {
return nil, err
}
// filter by the name provided
if request.(dao.ServiceRequest).NameRegex != "" {
services, err = filterByNameRegex(request.(dao.ServiceRequest).NameRegex, services)
if err != nil {
glog.Error("Facade.GetTaggedServices: err=", err)
return nil, err
}
}
// filter by the tenantID provided
if request.(dao.ServiceRequest).TenantID != "" {
services, err = f.filterByTenantID(ctx, request.(dao.ServiceRequest).TenantID, services)
if err != nil {
glog.Error("Facade.GetTaggedServices: err=", err)
return nil, err
}
}
return services, nil
default:
err := fmt.Errorf("Bad request type: %v", v)
glog.V(2).Info("Facade.GetTaggedServices: err=", err)
return nil, err
}
}
func (f *Facade) GetServices(ctx datastore.Context, request dao.EntityRequest) ([]service.Service, error) {
glog.V(3).Infof("Facade.GetServices")
store := f.serviceStore
var services []service.Service
var err error
if request.(dao.ServiceRequest).UpdatedSince != 0 {
services, err = store.GetUpdatedServices(ctx, request.(dao.ServiceRequest).UpdatedSince)
if err != nil {
glog.Error("Facade.GetServices: err=", err)
return nil, err
}
} else {
services, err = store.GetServices(ctx)
if err != nil {
glog.Error("Facade.GetServices: err=", err)
return nil, err
}
}
if err = f.fillOutServices(ctx, services); err != nil {
return nil, err
}
switch v := request.(type) {
case dao.ServiceRequest:
glog.V(3).Infof("request: %+v", request)
// filter by the name provided
if request.(dao.ServiceRequest).NameRegex != "" {
services, err = filterByNameRegex(request.(dao.ServiceRequest).NameRegex, services)
if err != nil {
glog.Error("Facade.GetTaggedServices: err=", err)
return nil, err
}
}
// filter by the tenantID provided
if request.(dao.ServiceRequest).TenantID != "" {
services, err = f.filterByTenantID(ctx, request.(dao.ServiceRequest).TenantID, services)
if err != nil {
glog.Error("Facade.GetTaggedServices: err=", err)
return nil, err
}
}
return services, nil
default:
err := fmt.Errorf("Bad request type %v: %+v", v, request)
glog.V(2).Info("Facade.GetTaggedServices: err=", err)
return nil, err
}
return services, nil
}
//restGetPool retrieves a Resource Pools. Response is ResourcePool
func restGetPool(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
poolID, err := url.QueryUnescape(r.PathParam("poolId"))
if err != nil {
restBadRequest(w, err)
return
}
client, err := ctx.getMasterClient()
if err != nil {
restServerError(w, err)
return
}
pool, err := client.GetResourcePool(poolID)
if err != nil {
glog.Error("Could not get resource pool: ", err)
restServerError(w, err)
return
}
hostIDs, err := getPoolHostIds(pool.ID, client)
if err != nil {
restServerError(w, err)
return
}
if err := buildPoolMonitoringProfile(pool, hostIDs, client); err != nil {
restServerError(w, err)
return
}
glog.V(4).Infof("restGetPool: id %s, pool %#v", poolID, pool)
w.WriteJson(&pool)
}
/*
registerIPs does the actual work of determining the IPs on the host. Parameters are the hostId for this host
and the function used to send the found IPs
*/
func registerIPs(hostId string, sendFn SendHostIPs) error {
interfaces, err := net.Interfaces()
if err != nil {
glog.Error("Problem reading interfaces: ", err)
return err
}
hostIPResources := make([]dao.HostIPResource, 0, len(interfaces))
for _, iface := range interfaces {
addrs, err := iface.Addrs()
if err != nil {
glog.Errorf("Problem reading address for interface %s: %s", iface.Name, err)
return err
}
for _, addr := range addrs {
//send address to Master
hostIp := dao.HostIPResource{}
hostIp.IPAddress = addr.String()
hostIp.InterfaceName = iface.Name
hostIPResources = append(hostIPResources, hostIp)
}
}
var unused int
glog.V(4).Infof("Agent registering IPs %v", hostIPResources)
hostIps := dao.HostIPs{}
hostIps.HostId = hostId
hostIps.IPs = hostIPResources
if err := sendFn(hostIps, &unused); err != nil {
glog.Errorf("Error registering IPs %v", err)
}
return nil
}
func buildPoolMonitoringProfile(pool *pool.ResourcePool, hostIDs []string, client *master.Client) error {
var totalMemory uint64 = 0
var totalCores int = 0
//calculate total memory and total cores
for i := range hostIDs {
host, err := client.GetHost(hostIDs[i])
if err != nil {
glog.Errorf("Failed to get host for id=%s -> %s", hostIDs[i], err)
return err
}
totalCores += host.Cores
totalMemory += host.Memory
}
tags := map[string][]string{"controlplane_host_id": hostIDs}
profile, err := hostPoolProfile.ReBuild("1h-ago", tags)
if err != nil {
glog.Error("Failed to create pool profile: %s", err)
return err
}
//add graphs to profile
profile.GraphConfigs = make([]domain.GraphConfig, 3)
profile.GraphConfigs[0] = newCpuConfigGraph(tags, totalCores)
profile.GraphConfigs[1] = newRSSConfigGraph(tags, totalMemory)
profile.GraphConfigs[2] = newMajorPageFaultGraph(tags)
pool.MonitoringProfile = *profile
return nil
}
//restGetPools retrieves all Resource Pools. Response is map[pool-id]ResourcePool
func restGetPools(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
client, err := ctx.getMasterClient()
if err != nil {
restServerError(w, err)
return
}
pools, err := client.GetResourcePools()
if err != nil {
glog.Error("Could not get resource pools: ", err)
restServerError(w, err)
return
}
poolsMap := make(map[string]*pool.ResourcePool)
for i, pool := range pools {
hostIDs, err := getPoolHostIds(pool.ID, client)
if err != nil {
restServerError(w, err)
return
}
if err := buildPoolMonitoringProfile(&pools[i], hostIDs, client); err != nil {
restServerError(w, err)
return
}
poolsMap[pool.ID] = &pools[i]
}
glog.V(4).Infof("restGetPools: pools %#v", poolsMap)
w.WriteJson(&poolsMap)
}
//restUpdatePool updates a resource pool. Request input is pool.ResourcePool
func restUpdatePool(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
poolID, err := url.QueryUnescape(r.PathParam("poolId"))
if err != nil {
restBadRequest(w, err)
return
}
var payload pool.ResourcePool
err = r.DecodeJsonPayload(&payload)
if err != nil {
glog.V(1).Info("Could not decode pool payload: ", err)
restBadRequest(w, err)
return
}
client, err := ctx.getMasterClient()
if err != nil {
restServerError(w, err)
return
}
err = client.UpdateResourcePool(payload)
if err != nil {
glog.Error("Unable to update pool: ", err)
restServerError(w, err)
return
}
glog.V(1).Info("Updated pool ", poolID)
w.WriteJson(&simpleResponse{"Updated resource pool", poolLinks(poolID)})
}
//restGetHost retrieves a host. Response is Host
func restGetHost(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
hostID, err := url.QueryUnescape(r.PathParam("hostId"))
if err != nil {
restBadRequest(w, err)
return
}
client, err := ctx.getMasterClient()
if err != nil {
restServerError(w, err)
return
}
host, err := client.GetHost(hostID)
if err != nil {
glog.Error("Could not get host: ", err)
restServerError(w, err)
return
}
if err := buildHostMonitoringProfile(host); err != nil {
restServerError(w, err)
return
}
glog.V(4).Infof("restGetHost: id %s, host %#v", hostID, host)
w.WriteJson(&host)
}
func DeleteNodebyData(path string, conn *zk.Conn, data []byte) error {
children, _, err := conn.Children(path)
if err != nil {
glog.Warning("Could not list children")
return err
}
for _, child := range children {
child_path := path + "/" + child
child_data, _, err := conn.Get(child_path)
if err != nil {
glog.Warning("Could not get data for %s", child_path)
continue
}
if bytes.Compare(data, child_data) == 0 {
for i := 0; i < 5; i++ {
_, stats, _ := conn.Get(child_path)
err = conn.Delete(child_path, stats.Version)
if err == nil || err == zk.ErrNoNode {
return nil
}
}
glog.Error("Could not delete matched node %s", child_path)
}
}
return nil
}
// Do the actual gathering and reporting of container statistics
func (sr StatsReporter) reportStats(t time.Time) {
glog.V(3).Infoln("Reporting container stats at: ", t)
memfiles := []string{"memory.stat"}
if err := filepath.Walk(MEMDIR, sr.mkReporter("memory", memfiles, t.Unix())); err != nil {
glog.Error("Problem reporting container memory statistics: ", err)
}
cpufiles := []string{"cpuacct.stat"}
if err := filepath.Walk(CPUDIR, sr.mkReporter("cpuacct", cpufiles, t.Unix())); err != nil {
glog.Error("Problem reporting container cpu statistics: ", err)
}
blkiofiles := []string{"blkio.sectors", "blkio.io_service_bytes", "blkio.io_serviced", "blkio.io_queued"}
if err := filepath.Walk(BLKIODIR, sr.mkReporter("blkio", blkiofiles, t.Unix())); err != nil {
glog.Error("Problem reporting container blkio statistics: ", err)
}
}
func watchService(cpDao dao.ControlPlane, conn *zk.Conn, shutdown <-chan int, done chan<- string, serviceId string) {
defer func() {
glog.V(3).Info("Exiting function watchService ", serviceId)
done <- serviceId
}()
for {
var service dao.Service
_, zkEvent, err := zzk.LoadServiceW(conn, serviceId, &service)
if err != nil {
glog.Errorf("Unable to load service %s: %v", serviceId, err)
return
}
_, _, childEvent, err := conn.ChildrenW(zzk.ServicePath(serviceId))
glog.V(1).Info("Leader watching for changes to service ", service.Name)
// check current state
var serviceStates []*dao.ServiceState
err = zzk.GetServiceStates(conn, &serviceStates, serviceId)
if err != nil {
glog.Error("Unable to retrieve running service states: ", err)
return
}
// Is the service supposed to be running at all?
switch {
case service.DesiredState == dao.SVC_STOP:
shutdownServiceInstances(conn, serviceStates, len(serviceStates))
case service.DesiredState == dao.SVC_RUN:
updateServiceInstances(cpDao, conn, &service, serviceStates)
default:
glog.Warningf("Unexpected desired state %d for service %s", service.DesiredState, service.Name)
}
select {
case evt := <-zkEvent:
if evt.Type == zk.EventNodeDeleted {
glog.V(0).Info("Shutting down due to node delete ", serviceId)
shutdownServiceInstances(conn, serviceStates, len(serviceStates))
return
}
glog.V(1).Infof("Service %s received event: %v", service.Name, evt)
continue
case evt := <-childEvent:
glog.V(1).Infof("Service %s received child event: %v", service.Name, evt)
continue
case <-shutdown:
glog.V(1).Info("Leader stopping watch on ", service.Name)
return
}
}
}
func RestGetPools(w *rest.ResponseWriter, r *rest.Request, client *serviced.ControlClient) {
var poolsMap map[string]*dao.ResourcePool
err := client.GetResourcePools(&empty, &poolsMap)
if err != nil {
glog.Error("Could not get resource pools: ", err)
RestServerError(w)
return
}
w.WriteJson(&poolsMap)
}
//getServices is an internal method that returns all Services without filling in all related service data like address assignments
//and modified config files
func (f *Facade) getServices(ctx datastore.Context) ([]service.Service, error) {
glog.V(3).Infof("Facade.GetServices")
store := f.serviceStore
results, err := store.GetServices(ctx)
if err != nil {
glog.Error("Facade.GetServices: err=", err)
return results, err
}
return results, nil
}
// GetServicesByPool looks up all services in a particular pool
func (f *Facade) GetServicesByPool(ctx datastore.Context, poolID string) ([]service.Service, error) {
glog.V(3).Infof("Facade.GetServicesByPool")
store := f.serviceStore
results, err := store.GetServicesByPool(ctx, poolID)
if err != nil {
glog.Error("Facade.GetServicesByPool: err=", err)
return results, err
}
if err = f.fillOutServices(ctx, results); err != nil {
return results, err
}
return results, nil
}
func RestRemovePool(w *rest.ResponseWriter, r *rest.Request, client *serviced.ControlClient) {
poolId, err := url.QueryUnescape(r.PathParam("poolId"))
if err != nil {
RestBadRequest(w)
return
}
var unused int
err = client.RemoveResourcePool(poolId, &unused)
if err != nil {
glog.Error("Could not remove resource pool: ", err)
RestServerError(w)
return
}
glog.V(0).Info("Removed pool ", poolId)
w.WriteJson(&SimpleResponse{"Removed resource pool", poolsLinks()})
}
// restServiceAutomaticAssignIP rest resource for automatic assigning ips to a service
func restServiceAutomaticAssignIP(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
serviceID, err := url.QueryUnescape(r.PathParam("serviceId"))
if err != nil {
glog.Errorf("Could not get serviceId: %v", err)
restBadRequest(w, err)
return
}
request := dao.AssignmentRequest{ServiceID: serviceID, IPAddress: "", AutoAssignment: true}
if err := client.AssignIPs(request, nil); err != nil {
glog.Error("Failed to automatically assign IPs: %+v -> %v", request, err)
restServerError(w, err)
return
}
restSuccess(w)
}
func RestDeployAppTemplate(w *rest.ResponseWriter, r *rest.Request, client *serviced.ControlClient) {
var payload dao.ServiceTemplateDeploymentRequest
err := r.DecodeJsonPayload(&payload)
if err != nil {
glog.V(1).Info("Could not decode deployment payload: ", err)
RestBadRequest(w)
return
}
var unused int
err = client.DeployTemplate(payload, &unused)
if err != nil {
glog.Error("Could not deploy template: ", err)
RestServerError(w)
return
}
glog.V(0).Info("Deployed template ", payload)
w.WriteJson(&SimpleResponse{"Deployed app template", servicesLinks()})
}
func RestAddPool(w *rest.ResponseWriter, r *rest.Request, client *serviced.ControlClient) {
var payload dao.ResourcePool
var poolId string
err := r.DecodeJsonPayload(&payload)
if err != nil {
glog.V(1).Info("Could not decode pool payload: ", err)
RestBadRequest(w)
return
}
err = client.AddResourcePool(payload, &poolId)
if err != nil {
glog.Error("Unable to add pool: ", err)
RestServerError(w)
return
}
glog.V(0).Info("Added pool ", poolId)
w.WriteJson(&SimpleResponse{"Added resource pool", poolLinks(poolId)})
}
//restRemovePool removes a resource pool using pool-id
func restRemovePool(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
poolID, err := url.QueryUnescape(r.PathParam("poolId"))
if err != nil {
restBadRequest(w, err)
return
}
client, err := ctx.getMasterClient()
if err != nil {
restServerError(w, err)
return
}
err = client.RemoveResourcePool(poolID)
if err != nil {
glog.Error("Could not remove resource pool: ", err)
restServerError(w, err)
return
}
glog.V(0).Info("Removed pool ", poolID)
w.WriteJson(&simpleResponse{"Removed resource pool", poolsLinks()})
}
func buildHostMonitoringProfile(host *host.Host) error {
tags := map[string][]string{"controlplane_host_id": []string{host.ID}}
profile, err := hostPoolProfile.ReBuild("1h-ago", tags)
if err != nil {
glog.Error("Failed to create host profile: %s", err)
return err
}
//add graphs to profile
profile.GraphConfigs = make([]domain.GraphConfig, 6)
profile.GraphConfigs[0] = newCpuConfigGraph(tags, host.Cores)
profile.GraphConfigs[1] = newLoadAverageGraph(tags)
profile.GraphConfigs[2] = newRSSConfigGraph(tags, host.Memory)
profile.GraphConfigs[3] = newOpenFileDescriptorsGraph(tags)
profile.GraphConfigs[4] = newMajorPageFaultGraph(tags)
profile.GraphConfigs[5] = newPagingGraph(tags)
host.MonitoringProfile = *profile
return nil
}
func createMuxListener() (net.Listener, error) {
if options.TLS {
glog.V(1).Info("using TLS on mux")
proxyCertPEM, proxyKeyPEM, err := getKeyPairs(options.CertPEMFile, options.KeyPEMFile)
if err != nil {
return nil, err
}
cert, err := tls.X509KeyPair([]byte(proxyCertPEM), []byte(proxyKeyPEM))
if err != nil {
glog.Error("ListenAndMux Error (tls.X509KeyPair): ", err)
return nil, err
}
tlsConfig := tls.Config{Certificates: []tls.Certificate{cert}}
glog.V(1).Infof("TLS enabled tcp mux listening on %d", options.MuxPort)
return tls.Listen("tcp", fmt.Sprintf(":%d", options.MuxPort), &tlsConfig)
}
return net.Listen("tcp", fmt.Sprintf(":%d", options.MuxPort))
}
func RestUpdatePool(w *rest.ResponseWriter, r *rest.Request, client *serviced.ControlClient) {
poolId, err := url.QueryUnescape(r.PathParam("poolId"))
if err != nil {
RestBadRequest(w)
return
}
var payload dao.ResourcePool
var unused int
err = r.DecodeJsonPayload(&payload)
if err != nil {
glog.V(1).Info("Could not decode pool payload: ", err)
RestBadRequest(w)
return
}
err = client.UpdateResourcePool(payload, &unused)
if err != nil {
glog.Error("Unable to update pool: ", err)
RestServerError(w)
return
}
glog.V(1).Info("Updated pool ", poolId)
w.WriteJson(&SimpleResponse{"Updated resource pool", poolLinks(poolId)})
}
func (f *Facade) deployServiceDefinitions(ctx datastore.Context, sds []servicedefinition.ServiceDefinition, pool string, parentServiceID string, volumes map[string]string, deploymentId string, tenantId *string) error {
// ensure that all images in the templates exist
imageIds := make(map[string]struct{})
for _, svc := range sds {
getSubServiceImageIDs(imageIds, svc)
}
for imageId, _ := range imageIds {
_, err := docker.FindImage(imageId, false)
if err != nil {
msg := fmt.Errorf("could not look up image %s: %s. Check your docker login and retry service deployment.", imageId, err)
glog.Error(err.Error())
return msg
}
}
for _, sd := range sds {
if _, err := f.deployServiceDefinition(ctx, sd, pool, parentServiceID, volumes, deploymentId, tenantId); err != nil {
return err
}
}
return nil
}
//restGetPoolIps retrieves a Resource Pools. Response is ResourcePool
func restGetPoolIps(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
poolID, err := url.QueryUnescape(r.PathParam("poolId"))
if err != nil {
restBadRequest(w, err)
return
}
client, err := ctx.getMasterClient()
if err != nil {
restServerError(w, err)
return
}
ips, err := client.GetPoolIPs(poolID)
if err != nil {
glog.Error("Could not get resource pool: ", err)
restServerError(w, err)
return
}
glog.V(4).Infof("restGetPoolIps: id %s, pool %#v", poolID, ips)
w.WriteJson(&ips)
}
//restAddPool add a resource pool. Request input is pool.ResourcePool
func restAddPool(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
var payload pool.ResourcePool
err := r.DecodeJsonPayload(&payload)
if err != nil {
glog.V(1).Info("Could not decode pool payload: ", err)
restBadRequest(w, err)
return
}
client, err := ctx.getMasterClient()
if err != nil {
restServerError(w, err)
return
}
err = client.AddResourcePool(payload)
if err != nil {
glog.Error("Unable to add pool: ", err)
restServerError(w, err)
return
}
glog.V(0).Info("Added pool ", payload.ID)
w.WriteJson(&simpleResponse{"Added resource pool", poolLinks(payload.ID)})
}
// Run executes the controller's main loop and block until the service exits
// according to it's restart policy or Close() is called.
func (c *Controller) Run() (err error) {
defer c.shutdown()
sigc := make(chan os.Signal, 1)
signal.Notify(sigc,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
env := os.Environ()
env = append(env, "CONTROLPLANE=1")
env = append(env, fmt.Sprintf("CONTROLPLANE_CONSUMER_URL=http://localhost%s/api/metrics/store", c.options.Metric.Address))
env = append(env, fmt.Sprintf("CONTROLPLANE_HOST_ID=%s", c.hostID))
env = append(env, fmt.Sprintf("CONTROLPLANE_TENANT_ID=%s", c.tenantID))
env = append(env, fmt.Sprintf("CONTROLPLANE_INSTANCE_ID=%s", c.options.Service.InstanceID))
env = append(env, fmt.Sprintf("CONTROLPLANE_SERVICED_ID=%s", c.options.Service.ID))
if err := writeEnvFile(env); err != nil {
return err
}
args := []string{"-c", "exec " + strings.Join(c.options.Service.Command, " ")}
startService := func() (*subprocess.Instance, chan error) {
service, serviceExited, _ := subprocess.New(time.Second*10, env, "/bin/sh", args...)
return service, serviceExited
}
sendSignal := func(service *subprocess.Instance, sig os.Signal) bool {
switch {
case c.PIDFile != "":
c.forwardSignal(sig)
case service != nil:
service.Notify(sig)
default:
return false
}
return true
}
rpcDead, err := c.rpcHealthCheck()
if err != nil {
glog.Error("Could not setup RPC ping check: %s", err)
return err
}
storageDead, err := c.storageHealthCheck()
if err != nil {
glog.Errorf("Could not set up storage check: %s", err)
return err
}
prereqsPassed := make(chan bool)
var startAfter <-chan time.Time
var exitAfter <-chan time.Time
var service *subprocess.Instance = nil
serviceExited := make(chan error, 1)
c.watchRemotePorts()
if err := c.handleControlCenterImports(rpcDead); err != nil {
glog.Error("Could not setup Control Center specific imports: ", err)
return err
}
go c.checkPrereqs(prereqsPassed, rpcDead)
go c.reapZombies(rpcDead)
healthExit := make(chan struct{})
defer close(healthExit)
c.kickOffHealthChecks(healthExit)
doRegisterEndpoints := true
exited := false
var shutdownService = func(service *subprocess.Instance, sig os.Signal) {
c.options.Service.Autorestart = false
if sendSignal(service, sig) {
sigc = nil
prereqsPassed = nil
startAfter = nil
rpcDead = nil
exitAfter = time.After(time.Second * 30)
close(healthExit)
} else {
c.exitStatus = 1
exited = true
}
}
var reregister <-chan struct{}
for !exited {
select {
case sig := <-sigc:
glog.Infof("Notifying subprocess of signal %v", sig)
shutdownService(service, sig)
case <-exitAfter:
glog.Infof("Killing unresponsive subprocess")
sendSignal(service, syscall.SIGKILL)
c.exitStatus = 1
exited = true
case <-prereqsPassed:
startAfter = time.After(time.Millisecond * 1)
prereqsPassed = nil
case exitError := <-serviceExited:
if !c.options.Service.Autorestart {
exitStatus, _ := utils.GetExitStatus(exitError)
if c.options.Logforwarder.Enabled {
time.Sleep(c.options.Logforwarder.SettleTime)
}
glog.Infof("Service Exited with status:%d due to %+v", exitStatus, exitError)
//set loop to end
exited = true
//exit with exit code, defer so that other cleanup can happen
c.exitStatus = exitStatus
} else {
glog.Infof("Restarting service process in 10 seconds.")
service = nil
startAfter = time.After(time.Second * 10)
}
case <-startAfter:
glog.Infof("Starting service process.")
service, serviceExited = startService()
if doRegisterEndpoints {
reregister = registerExportedEndpoints(c, rpcDead)
doRegisterEndpoints = false
}
startAfter = nil
case <-reregister:
reregister = registerExportedEndpoints(c, rpcDead)
case <-rpcDead:
glog.Infof("RPC Server has gone away, cleaning up")
shutdownService(service, syscall.SIGTERM)
case <-storageDead:
glog.Infof("Distributed storage for service %s has gone away; shutting down", c.options.Service.ID)
shutdownService(service, syscall.SIGTERM)
}
}
// Signal to health check registry that this instance is giving up the ghost.
client, err := node.NewLBClient(c.options.ServicedEndpoint)
if err != nil {
glog.Errorf("Could not create a client to endpoint: %s, %s", c.options.ServicedEndpoint, err)
return nil
}
defer client.Close()
c.Close()
var unused int
client.LogHealthCheck(domain.HealthCheckResult{c.options.Service.ID, c.options.Service.InstanceID, "__instance_shutdown", time.Now().String(), "passed"}, &unused)
return nil
}
func (s *scheduler) localSync(shutdown <-chan interface{}, rootConn client.Connection) {
wait := time.After(0)
retry:
for {
select {
case <-wait:
case <-shutdown:
return
}
pools, err := s.GetResourcePools()
if err != nil {
glog.Errorf("Could not get resource pools: %s", err)
wait = time.After(minWait)
continue
} else if err := zkservice.SyncResourcePools(rootConn, pools); err != nil {
glog.Errorf("Could not do a local sync of resource pools: %s", err)
wait = time.After(minWait)
continue
}
for _, pool := range pools {
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(pool.ID))
if err != nil {
glog.Errorf("Could not get a pool-based connection for %s to zookeeper: %s", pool.ID, err)
wait = time.After(minWait)
continue retry
}
// Update the hosts
if hosts, err := s.GetHostsByPool(pool.ID); err != nil {
glog.Errorf("Could not get hosts in pool %s: %s", pool.ID, err)
wait = time.After(minWait)
continue retry
} else if err := zkservice.SyncHosts(conn, hosts); err != nil {
glog.Errorf("Could not do a local sync of hosts: %s", err)
wait = time.After(minWait)
continue retry
}
// Update the services
if svcs, err := s.GetServicesByPool(pool.ID); err != nil {
glog.Errorf("Could not get services: %s", err)
wait = time.After(minWait)
continue retry
} else if zkservice.SyncServices(conn, svcs); err != nil {
glog.Error("Could not do a local sync of services: %s", err)
wait = time.After(minWait)
continue retry
}
// Update Virtual IPs
if err := zkvirtualips.SyncVirtualIPs(conn, pool.VirtualIPs); err != nil {
glog.Errorf("Could not sync virtual ips for %s: %s", pool.ID, err)
wait = time.After(minWait)
continue retry
}
}
wait = time.After(maxWait)
}
}
