Ejemplo n.º 1
0
func restDeployAppTemplate(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
	var payload dao.ServiceTemplateDeploymentRequest
	err := r.DecodeJsonPayload(&payload)
	if err != nil {
		glog.V(1).Info("Could not decode deployment payload: ", err)
		restBadRequest(w, err)
		return
	}
	var tenantID string
	err = client.DeployTemplate(payload, &tenantID)
	if err != nil {
		glog.Error("Could not deploy template: ", err)
		restServerError(w, err)
		return
	}
	glog.V(0).Info("Deployed template ", payload)

	assignmentRequest := dao.AssignmentRequest{tenantID, "", true}
	if err := client.AssignIPs(assignmentRequest, nil); err != nil {
		glog.Error("Could not automatically assign IPs: %v", err)
		return
	}

	glog.Infof("Automatically assigned IP addresses to service: %v", tenantID)
	// end of automatic IP assignment

	w.WriteJson(&simpleResponse{tenantID, servicesLinks()})
}
Ejemplo n.º 2
0
func (this *ControlPlaneDao) GetRunningServices(request dao.EntityRequest, allRunningServices *[]dao.RunningService) error {
	// we initialize the data container to something here in case it has not been initialized yet
	*allRunningServices = make([]dao.RunningService, 0)
	allPools, err := this.facade.GetResourcePools(datastore.Get())
	if err != nil {
		glog.Error("runningservice.go failed to get resource pool")
		return err
	} else if allPools == nil || len(allPools) == 0 {
		return fmt.Errorf("no resource pools found")
	}

	for _, aPool := range allPools {
		poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(aPool.ID))
		if err != nil {
			glog.Error("runningservice.go Failed to get connection based on pool: %v", aPool.ID)
			return err
		}

		singlePoolRunningServices := []dao.RunningService{}
		singlePoolRunningServices, err = zkservice.LoadRunningServices(poolBasedConn)
		if err != nil {
			glog.Errorf("Failed GetAllRunningServices: %v", err)
			return err
		}

		for _, rs := range singlePoolRunningServices {
			*allRunningServices = append(*allRunningServices, rs)
		}
	}

	return nil
}
Ejemplo n.º 3
0
// listenAndMux listens for incoming connections and attempts to multiplex them
// to the local service that they request via a Zen-Service header in their
// initial message.
func (mux *TCPMux) ListenAndMux() {
	var l net.Listener
	var err error

	if mux.UseTLS == false {
		l, err = net.Listen("tcp4", fmt.Sprintf(":%d", mux.Port))
	} else {
		cert, cerr := tls.X509KeyPair([]byte(proxyCertPEM), []byte(proxyKeyPEM))
		if cerr != nil {
			glog.Error("ListenAndMux Error (tls.X509KeyPair): ", cerr)
			return
		}

		tlsConfig := tls.Config{Certificates: []tls.Certificate{cert}}
		l, err = tls.Listen("tcp4", fmt.Sprintf(":%d", mux.Port), &tlsConfig)
	}
	if err != nil {
		glog.Error("ListenAndMux Error (net.Listen): ", err)
		return
	}
	defer l.Close()

	for {
		conn, err := l.Accept()
		if err != nil {
			glog.Error("ListenAndMux Error (net.Accept): ", err)
			return
		}

		go mux.MuxConnection(conn)
	}
}
Ejemplo n.º 4
0
func (f *Facade) GetTaggedServices(ctx datastore.Context, request dao.EntityRequest) ([]service.Service, error) {
	glog.V(3).Infof("Facade.GetTaggedServices")

	store := f.serviceStore
	switch v := request.(type) {
	case []string:
		results, err := store.GetTaggedServices(ctx, v...)
		if err != nil {
			glog.Error("Facade.GetTaggedServices: err=", err)
			return nil, err
		}
		if err = f.fillOutServices(ctx, results); err != nil {
			return nil, err
		}
		glog.V(2).Infof("Facade.GetTaggedServices: services=%v", results)
		return results, nil
	case dao.ServiceRequest:
		glog.V(3).Infof("request: %+v", request)

		// Get the tagged services
		services, err := store.GetTaggedServices(ctx, request.(dao.ServiceRequest).Tags...)
		if err != nil {
			glog.Error("Facade.GetTaggedServices: err=", err)
			return nil, err
		}
		if err = f.fillOutServices(ctx, services); err != nil {
			return nil, err
		}

		// filter by the name provided
		if request.(dao.ServiceRequest).NameRegex != "" {
			services, err = filterByNameRegex(request.(dao.ServiceRequest).NameRegex, services)
			if err != nil {
				glog.Error("Facade.GetTaggedServices: err=", err)
				return nil, err
			}
		}

		// filter by the tenantID provided
		if request.(dao.ServiceRequest).TenantID != "" {
			services, err = f.filterByTenantID(ctx, request.(dao.ServiceRequest).TenantID, services)
			if err != nil {
				glog.Error("Facade.GetTaggedServices: err=", err)
				return nil, err
			}
		}

		return services, nil
	default:
		err := fmt.Errorf("Bad request type: %v", v)
		glog.V(2).Info("Facade.GetTaggedServices: err=", err)
		return nil, err
	}
}
Ejemplo n.º 5
0
func (f *Facade) GetServices(ctx datastore.Context, request dao.EntityRequest) ([]service.Service, error) {
	glog.V(3).Infof("Facade.GetServices")
	store := f.serviceStore
	var services []service.Service
	var err error
	if request.(dao.ServiceRequest).UpdatedSince != 0 {
		services, err = store.GetUpdatedServices(ctx, request.(dao.ServiceRequest).UpdatedSince)
		if err != nil {
			glog.Error("Facade.GetServices: err=", err)
			return nil, err
		}
	} else {
		services, err = store.GetServices(ctx)
		if err != nil {
			glog.Error("Facade.GetServices: err=", err)
			return nil, err
		}
	}
	if err = f.fillOutServices(ctx, services); err != nil {
		return nil, err
	}

	switch v := request.(type) {
	case dao.ServiceRequest:
		glog.V(3).Infof("request: %+v", request)

		// filter by the name provided
		if request.(dao.ServiceRequest).NameRegex != "" {
			services, err = filterByNameRegex(request.(dao.ServiceRequest).NameRegex, services)
			if err != nil {
				glog.Error("Facade.GetTaggedServices: err=", err)
				return nil, err
			}
		}

		// filter by the tenantID provided
		if request.(dao.ServiceRequest).TenantID != "" {
			services, err = f.filterByTenantID(ctx, request.(dao.ServiceRequest).TenantID, services)
			if err != nil {
				glog.Error("Facade.GetTaggedServices: err=", err)
				return nil, err
			}
		}

		return services, nil
	default:
		err := fmt.Errorf("Bad request type %v: %+v", v, request)
		glog.V(2).Info("Facade.GetTaggedServices: err=", err)
		return nil, err
	}
	return services, nil
}
Ejemplo n.º 6
0
//restGetPool retrieves a Resource Pools. Response is ResourcePool
func restGetPool(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	poolID, err := url.QueryUnescape(r.PathParam("poolId"))
	if err != nil {
		restBadRequest(w, err)
		return
	}

	client, err := ctx.getMasterClient()
	if err != nil {
		restServerError(w, err)
		return
	}

	pool, err := client.GetResourcePool(poolID)
	if err != nil {
		glog.Error("Could not get resource pool: ", err)
		restServerError(w, err)
		return
	}

	hostIDs, err := getPoolHostIds(pool.ID, client)
	if err != nil {
		restServerError(w, err)
		return
	}

	if err := buildPoolMonitoringProfile(pool, hostIDs, client); err != nil {
		restServerError(w, err)
		return
	}

	glog.V(4).Infof("restGetPool: id %s, pool %#v", poolID, pool)
	w.WriteJson(&pool)
}
Ejemplo n.º 7
0
/*
registerIPs does the actual work of determining the IPs on the host. Parameters are the hostId for this host
and the function used to send the found IPs
*/
func registerIPs(hostId string, sendFn SendHostIPs) error {

	interfaces, err := net.Interfaces()
	if err != nil {
		glog.Error("Problem reading interfaces: ", err)
		return err
	}
	hostIPResources := make([]dao.HostIPResource, 0, len(interfaces))
	for _, iface := range interfaces {
		addrs, err := iface.Addrs()
		if err != nil {
			glog.Errorf("Problem reading address for interface %s: %s", iface.Name, err)
			return err
		}
		for _, addr := range addrs {
			//send address to Master
			hostIp := dao.HostIPResource{}
			hostIp.IPAddress = addr.String()
			hostIp.InterfaceName = iface.Name
			hostIPResources = append(hostIPResources, hostIp)
		}
	}
	var unused int
	glog.V(4).Infof("Agent registering IPs %v", hostIPResources)
	hostIps := dao.HostIPs{}
	hostIps.HostId = hostId
	hostIps.IPs = hostIPResources
	if err := sendFn(hostIps, &unused); err != nil {
		glog.Errorf("Error registering IPs %v", err)
	}
	return nil
}
Ejemplo n.º 8
0
func buildPoolMonitoringProfile(pool *pool.ResourcePool, hostIDs []string, client *master.Client) error {
	var totalMemory uint64 = 0
	var totalCores int = 0

	//calculate total memory and total cores
	for i := range hostIDs {
		host, err := client.GetHost(hostIDs[i])
		if err != nil {
			glog.Errorf("Failed to get host for id=%s -> %s", hostIDs[i], err)
			return err
		}

		totalCores += host.Cores
		totalMemory += host.Memory
	}

	tags := map[string][]string{"controlplane_host_id": hostIDs}
	profile, err := hostPoolProfile.ReBuild("1h-ago", tags)
	if err != nil {
		glog.Error("Failed to create pool profile: %s", err)
		return err
	}

	//add graphs to profile
	profile.GraphConfigs = make([]domain.GraphConfig, 3)
	profile.GraphConfigs[0] = newCpuConfigGraph(tags, totalCores)
	profile.GraphConfigs[1] = newRSSConfigGraph(tags, totalMemory)
	profile.GraphConfigs[2] = newMajorPageFaultGraph(tags)

	pool.MonitoringProfile = *profile
	return nil
}
Ejemplo n.º 9
0
//restGetPools retrieves all Resource Pools. Response is map[pool-id]ResourcePool
func restGetPools(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	client, err := ctx.getMasterClient()
	if err != nil {
		restServerError(w, err)
		return
	}

	pools, err := client.GetResourcePools()
	if err != nil {
		glog.Error("Could not get resource pools: ", err)
		restServerError(w, err)
		return
	}
	poolsMap := make(map[string]*pool.ResourcePool)
	for i, pool := range pools {
		hostIDs, err := getPoolHostIds(pool.ID, client)
		if err != nil {
			restServerError(w, err)
			return
		}

		if err := buildPoolMonitoringProfile(&pools[i], hostIDs, client); err != nil {
			restServerError(w, err)
			return
		}

		poolsMap[pool.ID] = &pools[i]
	}
	glog.V(4).Infof("restGetPools: pools %#v", poolsMap)
	w.WriteJson(&poolsMap)
}
Ejemplo n.º 10
0
//restUpdatePool updates a resource pool. Request input is pool.ResourcePool
func restUpdatePool(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	poolID, err := url.QueryUnescape(r.PathParam("poolId"))
	if err != nil {
		restBadRequest(w, err)
		return
	}
	var payload pool.ResourcePool
	err = r.DecodeJsonPayload(&payload)
	if err != nil {
		glog.V(1).Info("Could not decode pool payload: ", err)
		restBadRequest(w, err)
		return
	}
	client, err := ctx.getMasterClient()
	if err != nil {
		restServerError(w, err)
		return
	}
	err = client.UpdateResourcePool(payload)
	if err != nil {
		glog.Error("Unable to update pool: ", err)
		restServerError(w, err)
		return
	}
	glog.V(1).Info("Updated pool ", poolID)
	w.WriteJson(&simpleResponse{"Updated resource pool", poolLinks(poolID)})
}
Ejemplo n.º 11
0
//restGetHost retrieves a host. Response is Host
func restGetHost(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	hostID, err := url.QueryUnescape(r.PathParam("hostId"))
	if err != nil {
		restBadRequest(w, err)
		return
	}

	client, err := ctx.getMasterClient()
	if err != nil {
		restServerError(w, err)
		return
	}

	host, err := client.GetHost(hostID)
	if err != nil {
		glog.Error("Could not get host: ", err)
		restServerError(w, err)
		return
	}

	if err := buildHostMonitoringProfile(host); err != nil {
		restServerError(w, err)
		return
	}

	glog.V(4).Infof("restGetHost: id %s, host %#v", hostID, host)
	w.WriteJson(&host)
}
Ejemplo n.º 12
0
func DeleteNodebyData(path string, conn *zk.Conn, data []byte) error {

	children, _, err := conn.Children(path)
	if err != nil {
		glog.Warning("Could not list children")
		return err
	}
	for _, child := range children {
		child_path := path + "/" + child
		child_data, _, err := conn.Get(child_path)
		if err != nil {
			glog.Warning("Could not get data for %s", child_path)
			continue
		}
		if bytes.Compare(data, child_data) == 0 {
			for i := 0; i < 5; i++ {
				_, stats, _ := conn.Get(child_path)
				err = conn.Delete(child_path, stats.Version)
				if err == nil || err == zk.ErrNoNode {
					return nil
				}
			}
			glog.Error("Could not delete matched node %s", child_path)
		}
	}
	return nil
}
Ejemplo n.º 13
0
// Do the actual gathering and reporting of container statistics
func (sr StatsReporter) reportStats(t time.Time) {
	glog.V(3).Infoln("Reporting container stats at: ", t)

	memfiles := []string{"memory.stat"}
	if err := filepath.Walk(MEMDIR, sr.mkReporter("memory", memfiles, t.Unix())); err != nil {
		glog.Error("Problem reporting container memory statistics: ", err)
	}

	cpufiles := []string{"cpuacct.stat"}
	if err := filepath.Walk(CPUDIR, sr.mkReporter("cpuacct", cpufiles, t.Unix())); err != nil {
		glog.Error("Problem reporting container cpu statistics: ", err)
	}

	blkiofiles := []string{"blkio.sectors", "blkio.io_service_bytes", "blkio.io_serviced", "blkio.io_queued"}
	if err := filepath.Walk(BLKIODIR, sr.mkReporter("blkio", blkiofiles, t.Unix())); err != nil {
		glog.Error("Problem reporting container blkio statistics: ", err)
	}
}
Ejemplo n.º 14
0
func watchService(cpDao dao.ControlPlane, conn *zk.Conn, shutdown <-chan int, done chan<- string, serviceId string) {
	defer func() {
		glog.V(3).Info("Exiting function watchService ", serviceId)
		done <- serviceId
	}()
	for {
		var service dao.Service
		_, zkEvent, err := zzk.LoadServiceW(conn, serviceId, &service)
		if err != nil {
			glog.Errorf("Unable to load service %s: %v", serviceId, err)
			return
		}
		_, _, childEvent, err := conn.ChildrenW(zzk.ServicePath(serviceId))

		glog.V(1).Info("Leader watching for changes to service ", service.Name)

		// check current state
		var serviceStates []*dao.ServiceState
		err = zzk.GetServiceStates(conn, &serviceStates, serviceId)
		if err != nil {
			glog.Error("Unable to retrieve running service states: ", err)
			return
		}

		// Is the service supposed to be running at all?
		switch {
		case service.DesiredState == dao.SVC_STOP:
			shutdownServiceInstances(conn, serviceStates, len(serviceStates))
		case service.DesiredState == dao.SVC_RUN:
			updateServiceInstances(cpDao, conn, &service, serviceStates)
		default:
			glog.Warningf("Unexpected desired state %d for service %s", service.DesiredState, service.Name)
		}

		select {
		case evt := <-zkEvent:
			if evt.Type == zk.EventNodeDeleted {
				glog.V(0).Info("Shutting down due to node delete ", serviceId)
				shutdownServiceInstances(conn, serviceStates, len(serviceStates))
				return
			}
			glog.V(1).Infof("Service %s received event: %v", service.Name, evt)
			continue

		case evt := <-childEvent:
			glog.V(1).Infof("Service %s received child event: %v", service.Name, evt)
			continue

		case <-shutdown:
			glog.V(1).Info("Leader stopping watch on ", service.Name)
			return

		}
	}

}
Ejemplo n.º 15
0
func RestGetPools(w *rest.ResponseWriter, r *rest.Request, client *serviced.ControlClient) {
	var poolsMap map[string]*dao.ResourcePool
	err := client.GetResourcePools(&empty, &poolsMap)
	if err != nil {
		glog.Error("Could not get resource pools: ", err)
		RestServerError(w)
		return
	}
	w.WriteJson(&poolsMap)
}
Ejemplo n.º 16
0
//getServices is an internal method that returns all Services without filling in all related service data like address assignments
//and modified config files
func (f *Facade) getServices(ctx datastore.Context) ([]service.Service, error) {
	glog.V(3).Infof("Facade.GetServices")
	store := f.serviceStore
	results, err := store.GetServices(ctx)
	if err != nil {
		glog.Error("Facade.GetServices: err=", err)
		return results, err
	}
	return results, nil
}
Ejemplo n.º 17
0
// GetServicesByPool looks up all services in a particular pool
func (f *Facade) GetServicesByPool(ctx datastore.Context, poolID string) ([]service.Service, error) {
	glog.V(3).Infof("Facade.GetServicesByPool")
	store := f.serviceStore
	results, err := store.GetServicesByPool(ctx, poolID)
	if err != nil {
		glog.Error("Facade.GetServicesByPool: err=", err)
		return results, err
	}
	if err = f.fillOutServices(ctx, results); err != nil {
		return results, err
	}
	return results, nil
}
Ejemplo n.º 18
0
func RestRemovePool(w *rest.ResponseWriter, r *rest.Request, client *serviced.ControlClient) {
	poolId, err := url.QueryUnescape(r.PathParam("poolId"))
	if err != nil {
		RestBadRequest(w)
		return
	}
	var unused int
	err = client.RemoveResourcePool(poolId, &unused)
	if err != nil {
		glog.Error("Could not remove resource pool: ", err)
		RestServerError(w)
		return
	}
	glog.V(0).Info("Removed pool ", poolId)
	w.WriteJson(&SimpleResponse{"Removed resource pool", poolsLinks()})
}
Ejemplo n.º 19
0
// restServiceAutomaticAssignIP rest resource for automatic assigning ips to a service
func restServiceAutomaticAssignIP(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
	serviceID, err := url.QueryUnescape(r.PathParam("serviceId"))
	if err != nil {
		glog.Errorf("Could not get serviceId: %v", err)
		restBadRequest(w, err)
		return
	}

	request := dao.AssignmentRequest{ServiceID: serviceID, IPAddress: "", AutoAssignment: true}
	if err := client.AssignIPs(request, nil); err != nil {
		glog.Error("Failed to automatically assign IPs: %+v -> %v", request, err)
		restServerError(w, err)
		return
	}

	restSuccess(w)
}
Ejemplo n.º 20
0
func RestDeployAppTemplate(w *rest.ResponseWriter, r *rest.Request, client *serviced.ControlClient) {
	var payload dao.ServiceTemplateDeploymentRequest
	err := r.DecodeJsonPayload(&payload)
	if err != nil {
		glog.V(1).Info("Could not decode deployment payload: ", err)
		RestBadRequest(w)
		return
	}
	var unused int
	err = client.DeployTemplate(payload, &unused)
	if err != nil {
		glog.Error("Could not deploy template: ", err)
		RestServerError(w)
		return
	}
	glog.V(0).Info("Deployed template ", payload)
	w.WriteJson(&SimpleResponse{"Deployed app template", servicesLinks()})
}
Ejemplo n.º 21
0
func RestAddPool(w *rest.ResponseWriter, r *rest.Request, client *serviced.ControlClient) {
	var payload dao.ResourcePool
	var poolId string
	err := r.DecodeJsonPayload(&payload)
	if err != nil {
		glog.V(1).Info("Could not decode pool payload: ", err)
		RestBadRequest(w)
		return
	}
	err = client.AddResourcePool(payload, &poolId)
	if err != nil {
		glog.Error("Unable to add pool: ", err)
		RestServerError(w)
		return
	}
	glog.V(0).Info("Added pool ", poolId)
	w.WriteJson(&SimpleResponse{"Added resource pool", poolLinks(poolId)})
}
Ejemplo n.º 22
0
//restRemovePool removes a resource pool using pool-id
func restRemovePool(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	poolID, err := url.QueryUnescape(r.PathParam("poolId"))
	if err != nil {
		restBadRequest(w, err)
		return
	}
	client, err := ctx.getMasterClient()
	if err != nil {
		restServerError(w, err)
		return
	}
	err = client.RemoveResourcePool(poolID)
	if err != nil {
		glog.Error("Could not remove resource pool: ", err)
		restServerError(w, err)
		return
	}
	glog.V(0).Info("Removed pool ", poolID)
	w.WriteJson(&simpleResponse{"Removed resource pool", poolsLinks()})
}
Ejemplo n.º 23
0
func buildHostMonitoringProfile(host *host.Host) error {
	tags := map[string][]string{"controlplane_host_id": []string{host.ID}}
	profile, err := hostPoolProfile.ReBuild("1h-ago", tags)
	if err != nil {
		glog.Error("Failed to create host profile: %s", err)
		return err
	}

	//add graphs to profile
	profile.GraphConfigs = make([]domain.GraphConfig, 6)
	profile.GraphConfigs[0] = newCpuConfigGraph(tags, host.Cores)
	profile.GraphConfigs[1] = newLoadAverageGraph(tags)
	profile.GraphConfigs[2] = newRSSConfigGraph(tags, host.Memory)
	profile.GraphConfigs[3] = newOpenFileDescriptorsGraph(tags)
	profile.GraphConfigs[4] = newMajorPageFaultGraph(tags)
	profile.GraphConfigs[5] = newPagingGraph(tags)

	host.MonitoringProfile = *profile
	return nil
}
Ejemplo n.º 24
0
func createMuxListener() (net.Listener, error) {
	if options.TLS {
		glog.V(1).Info("using TLS on mux")

		proxyCertPEM, proxyKeyPEM, err := getKeyPairs(options.CertPEMFile, options.KeyPEMFile)
		if err != nil {
			return nil, err
		}

		cert, err := tls.X509KeyPair([]byte(proxyCertPEM), []byte(proxyKeyPEM))
		if err != nil {
			glog.Error("ListenAndMux Error (tls.X509KeyPair): ", err)
			return nil, err
		}

		tlsConfig := tls.Config{Certificates: []tls.Certificate{cert}}
		glog.V(1).Infof("TLS enabled tcp mux listening on %d", options.MuxPort)
		return tls.Listen("tcp", fmt.Sprintf(":%d", options.MuxPort), &tlsConfig)

	}
	return net.Listen("tcp", fmt.Sprintf(":%d", options.MuxPort))
}
Ejemplo n.º 25
0
func RestUpdatePool(w *rest.ResponseWriter, r *rest.Request, client *serviced.ControlClient) {
	poolId, err := url.QueryUnescape(r.PathParam("poolId"))
	if err != nil {
		RestBadRequest(w)
		return
	}
	var payload dao.ResourcePool
	var unused int
	err = r.DecodeJsonPayload(&payload)
	if err != nil {
		glog.V(1).Info("Could not decode pool payload: ", err)
		RestBadRequest(w)
		return
	}
	err = client.UpdateResourcePool(payload, &unused)
	if err != nil {
		glog.Error("Unable to update pool: ", err)
		RestServerError(w)
		return
	}
	glog.V(1).Info("Updated pool ", poolId)
	w.WriteJson(&SimpleResponse{"Updated resource pool", poolLinks(poolId)})
}
Ejemplo n.º 26
0
func (f *Facade) deployServiceDefinitions(ctx datastore.Context, sds []servicedefinition.ServiceDefinition, pool string, parentServiceID string, volumes map[string]string, deploymentId string, tenantId *string) error {
	// ensure that all images in the templates exist
	imageIds := make(map[string]struct{})
	for _, svc := range sds {
		getSubServiceImageIDs(imageIds, svc)
	}

	for imageId, _ := range imageIds {
		_, err := docker.FindImage(imageId, false)
		if err != nil {
			msg := fmt.Errorf("could not look up image %s: %s. Check your docker login and retry service deployment.", imageId, err)
			glog.Error(err.Error())
			return msg
		}
	}

	for _, sd := range sds {
		if _, err := f.deployServiceDefinition(ctx, sd, pool, parentServiceID, volumes, deploymentId, tenantId); err != nil {
			return err
		}
	}
	return nil
}
Ejemplo n.º 27
0
//restGetPoolIps retrieves a Resource Pools. Response is ResourcePool
func restGetPoolIps(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	poolID, err := url.QueryUnescape(r.PathParam("poolId"))
	if err != nil {
		restBadRequest(w, err)
		return
	}

	client, err := ctx.getMasterClient()
	if err != nil {
		restServerError(w, err)
		return
	}

	ips, err := client.GetPoolIPs(poolID)
	if err != nil {
		glog.Error("Could not get resource pool: ", err)
		restServerError(w, err)
		return
	}

	glog.V(4).Infof("restGetPoolIps: id %s, pool %#v", poolID, ips)
	w.WriteJson(&ips)
}
Ejemplo n.º 28
0
//restAddPool add a resource pool. Request input is pool.ResourcePool
func restAddPool(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	var payload pool.ResourcePool
	err := r.DecodeJsonPayload(&payload)
	if err != nil {
		glog.V(1).Info("Could not decode pool payload: ", err)
		restBadRequest(w, err)
		return
	}
	client, err := ctx.getMasterClient()
	if err != nil {
		restServerError(w, err)
		return
	}

	err = client.AddResourcePool(payload)
	if err != nil {
		glog.Error("Unable to add pool: ", err)
		restServerError(w, err)
		return
	}
	glog.V(0).Info("Added pool ", payload.ID)
	w.WriteJson(&simpleResponse{"Added resource pool", poolLinks(payload.ID)})
}
Ejemplo n.º 29
0
// Run executes the controller's main loop and block until the service exits
// according to it's restart policy or Close() is called.
func (c *Controller) Run() (err error) {
	defer c.shutdown()
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	env := os.Environ()
	env = append(env, "CONTROLPLANE=1")
	env = append(env, fmt.Sprintf("CONTROLPLANE_CONSUMER_URL=http://localhost%s/api/metrics/store", c.options.Metric.Address))
	env = append(env, fmt.Sprintf("CONTROLPLANE_HOST_ID=%s", c.hostID))
	env = append(env, fmt.Sprintf("CONTROLPLANE_TENANT_ID=%s", c.tenantID))
	env = append(env, fmt.Sprintf("CONTROLPLANE_INSTANCE_ID=%s", c.options.Service.InstanceID))
	env = append(env, fmt.Sprintf("CONTROLPLANE_SERVICED_ID=%s", c.options.Service.ID))

	if err := writeEnvFile(env); err != nil {
		return err
	}

	args := []string{"-c", "exec " + strings.Join(c.options.Service.Command, " ")}

	startService := func() (*subprocess.Instance, chan error) {
		service, serviceExited, _ := subprocess.New(time.Second*10, env, "/bin/sh", args...)
		return service, serviceExited
	}

	sendSignal := func(service *subprocess.Instance, sig os.Signal) bool {
		switch {
		case c.PIDFile != "":
			c.forwardSignal(sig)
		case service != nil:
			service.Notify(sig)
		default:
			return false
		}
		return true
	}

	rpcDead, err := c.rpcHealthCheck()
	if err != nil {
		glog.Error("Could not setup RPC ping check: %s", err)
		return err
	}

	storageDead, err := c.storageHealthCheck()
	if err != nil {
		glog.Errorf("Could not set up storage check: %s", err)
		return err
	}

	prereqsPassed := make(chan bool)
	var startAfter <-chan time.Time
	var exitAfter <-chan time.Time
	var service *subprocess.Instance = nil
	serviceExited := make(chan error, 1)
	c.watchRemotePorts()
	if err := c.handleControlCenterImports(rpcDead); err != nil {
		glog.Error("Could not setup Control Center specific imports: ", err)
		return err
	}
	go c.checkPrereqs(prereqsPassed, rpcDead)
	go c.reapZombies(rpcDead)
	healthExit := make(chan struct{})
	defer close(healthExit)
	c.kickOffHealthChecks(healthExit)
	doRegisterEndpoints := true
	exited := false

	var shutdownService = func(service *subprocess.Instance, sig os.Signal) {
		c.options.Service.Autorestart = false
		if sendSignal(service, sig) {
			sigc = nil
			prereqsPassed = nil
			startAfter = nil
			rpcDead = nil
			exitAfter = time.After(time.Second * 30)
			close(healthExit)
		} else {
			c.exitStatus = 1
			exited = true
		}
	}

	var reregister <-chan struct{}

	for !exited {
		select {
		case sig := <-sigc:
			glog.Infof("Notifying subprocess of signal %v", sig)
			shutdownService(service, sig)

		case <-exitAfter:
			glog.Infof("Killing unresponsive subprocess")
			sendSignal(service, syscall.SIGKILL)
			c.exitStatus = 1
			exited = true

		case <-prereqsPassed:
			startAfter = time.After(time.Millisecond * 1)
			prereqsPassed = nil

		case exitError := <-serviceExited:
			if !c.options.Service.Autorestart {
				exitStatus, _ := utils.GetExitStatus(exitError)
				if c.options.Logforwarder.Enabled {
					time.Sleep(c.options.Logforwarder.SettleTime)
				}
				glog.Infof("Service Exited with status:%d due to %+v", exitStatus, exitError)
				//set loop to end
				exited = true
				//exit with exit code, defer so that other cleanup can happen
				c.exitStatus = exitStatus

			} else {
				glog.Infof("Restarting service process in 10 seconds.")
				service = nil
				startAfter = time.After(time.Second * 10)
			}

		case <-startAfter:
			glog.Infof("Starting service process.")
			service, serviceExited = startService()
			if doRegisterEndpoints {
				reregister = registerExportedEndpoints(c, rpcDead)
				doRegisterEndpoints = false
			}
			startAfter = nil
		case <-reregister:
			reregister = registerExportedEndpoints(c, rpcDead)
		case <-rpcDead:
			glog.Infof("RPC Server has gone away, cleaning up")
			shutdownService(service, syscall.SIGTERM)
		case <-storageDead:
			glog.Infof("Distributed storage for service %s has gone away; shutting down", c.options.Service.ID)
			shutdownService(service, syscall.SIGTERM)
		}
	}
	// Signal to health check registry that this instance is giving up the ghost.
	client, err := node.NewLBClient(c.options.ServicedEndpoint)
	if err != nil {
		glog.Errorf("Could not create a client to endpoint: %s, %s", c.options.ServicedEndpoint, err)
		return nil
	}
	defer client.Close()
	c.Close()
	var unused int
	client.LogHealthCheck(domain.HealthCheckResult{c.options.Service.ID, c.options.Service.InstanceID, "__instance_shutdown", time.Now().String(), "passed"}, &unused)
	return nil
}
Ejemplo n.º 30
0
func (s *scheduler) localSync(shutdown <-chan interface{}, rootConn client.Connection) {
	wait := time.After(0)

retry:
	for {
		select {
		case <-wait:
		case <-shutdown:
			return
		}

		pools, err := s.GetResourcePools()
		if err != nil {
			glog.Errorf("Could not get resource pools: %s", err)
			wait = time.After(minWait)
			continue
		} else if err := zkservice.SyncResourcePools(rootConn, pools); err != nil {
			glog.Errorf("Could not do a local sync of resource pools: %s", err)
			wait = time.After(minWait)
			continue
		}

		for _, pool := range pools {
			conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(pool.ID))
			if err != nil {
				glog.Errorf("Could not get a pool-based connection for %s to zookeeper: %s", pool.ID, err)
				wait = time.After(minWait)
				continue retry
			}

			// Update the hosts
			if hosts, err := s.GetHostsByPool(pool.ID); err != nil {
				glog.Errorf("Could not get hosts in pool %s: %s", pool.ID, err)
				wait = time.After(minWait)
				continue retry
			} else if err := zkservice.SyncHosts(conn, hosts); err != nil {
				glog.Errorf("Could not do a local sync of hosts: %s", err)
				wait = time.After(minWait)
				continue retry
			}

			// Update the services
			if svcs, err := s.GetServicesByPool(pool.ID); err != nil {
				glog.Errorf("Could not get services: %s", err)
				wait = time.After(minWait)
				continue retry
			} else if zkservice.SyncServices(conn, svcs); err != nil {
				glog.Error("Could not do a local sync of services: %s", err)
				wait = time.After(minWait)
				continue retry
			}

			// Update Virtual IPs
			if err := zkvirtualips.SyncVirtualIPs(conn, pool.VirtualIPs); err != nil {
				glog.Errorf("Could not sync virtual ips for %s: %s", pool.ID, err)
				wait = time.After(minWait)
				continue retry
			}
		}

		wait = time.After(maxWait)
	}
}