func (this *ControlPlaneDao) GetServiceLogs(serviceID string, logs *string) error {
glog.V(3).Info("ControlPlaneDao.GetServiceLogs serviceID=", serviceID)
var serviceStates []servicestate.ServiceState
if err := this.GetServiceStates(serviceID, &serviceStates); err != nil {
glog.Errorf("ControlPlaneDao.GetServiceLogs failed: %v", err)
return err
}
if len(serviceStates) == 0 {
glog.V(1).Info("Unable to find any running services for service:", serviceID)
return nil
}
serviceState := serviceStates[0]
endpoint := fmt.Sprintf("%s:%d", serviceState.HostIP, this.rpcPort)
agentClient, err := agent.NewClient(endpoint)
if err != nil {
glog.Errorf("could not create client to %s", endpoint)
return err
}
defer agentClient.Close()
if mylogs, err := agentClient.GetDockerLogs(serviceState.DockerID); err != nil {
glog.Errorf("could not get docker logs from agent client: %s", err)
return err
} else {
*logs = mylogs
}
return nil
}
func (f *Facade) RemoveService(ctx datastore.Context, id string) error {
store := f.serviceStore
return f.walkServices(ctx, id, true, func(svc *service.Service) error {
// remove all address assignments
for _, endpoint := range svc.Endpoints {
if assignment, err := f.FindAssignmentByServiceEndpoint(ctx, svc.ID, endpoint.Name); err != nil {
glog.Errorf("Could not find address assignment %s for service %s (%s): %s", endpoint.Name, svc.Name, svc.ID, err)
return err
} else if assignment != nil {
if err := f.RemoveAddressAssignment(ctx, assignment.ID); err != nil {
glog.Errorf("Could not remove address assignment %s from service %s (%s): %s", endpoint.Name, svc.Name, svc.ID, err)
return err
}
}
endpoint.RemoveAssignment()
}
if err := zkAPI(f).RemoveService(svc); err != nil {
glog.Errorf("Could not remove service %s (%s) from zookeeper: %s", svc.Name, svc.ID, err)
return err
}
if err := store.Delete(ctx, svc.ID); err != nil {
glog.Errorf("Error while removing service %s (%s): %s", svc.Name, svc.ID, err)
return err
}
return nil
})
}
//ValidEntity validates Host fields
func (h *Host) ValidEntity() error {
glog.V(4).Info("Validating host")
//if err := validation.ValidHostID(entity.ID); err != nil {
// return fmt.Errorf("invalid hostid:'%s' for host Name:'%s' IP:%s", entity.ID, entity.Name, entity.IPAddr)
//}
trimmedID := strings.TrimSpace(h.ID)
violations := validation.NewValidationError()
violations.Add(validation.NotEmpty("Host.ID", h.ID))
violations.Add(validation.ValidHostID(h.ID))
violations.Add(validation.StringsEqual(h.ID, trimmedID, "leading and trailing spaces not allowed for host id"))
violations.Add(validation.ValidPort(h.RPCPort))
violations.Add(validation.NotEmpty("Host.PoolID", h.PoolID))
violations.Add(validation.IsIP(h.IPAddr))
//TODO: what should we be validating here? It doesn't seem to work for
glog.V(4).Infof("Validating IPAddr %v for host %s", h.IPAddr, h.ID)
ipAddr, err := net.ResolveIPAddr("ip4", h.IPAddr)
if err != nil {
glog.Errorf("Could not resolve: %s to an ip4 address: %v", h.IPAddr, err)
violations.Add(err)
} else if ipAddr.IP.IsLoopback() {
glog.Errorf("Can not use %s as host address because it is a loopback address", h.IPAddr)
violations.Add(errors.New("host ip can not be a loopback address"))
}
if len(violations.Errors) > 0 {
return violations
}
return nil
}
// getServiceState gets the service state for a serviceID matching the instance ID specified
func getServiceState(conn coordclient.Connection, serviceID, instanceIDStr string) (*servicestate.ServiceState, error) {
tmpID, err := strconv.Atoi(instanceIDStr)
if err != nil {
glog.Errorf("Unable to interpret InstanceID: %s", instanceIDStr)
return nil, fmt.Errorf("endpoint.go getServiceState failed: %v", err)
}
instanceID := int(tmpID)
for {
serviceStates, err := getServiceStates(conn, serviceID)
if err != nil {
glog.Errorf("Unable to retrieve running service (%s) states: %v", serviceID, err)
return nil, fmt.Errorf("endpoint.go getServiceState zzk.GetServiceStates failed: %v", err)
}
for ii, ss := range serviceStates {
if ss.InstanceID == instanceID && ss.PrivateIP != "" {
return &serviceStates[ii], nil
}
}
glog.V(2).Infof("Polling to retrieve service state instanceID:%d with valid PrivateIP", instanceID)
time.Sleep(1 * time.Second)
}
return nil, fmt.Errorf("unable to retrieve service state")
}
// processTenantEndpoint updates the addresses for an imported endpoint
func (c *Controller) processTenantEndpoint(conn coordclient.Connection, parentPath string, hostContainerIDs ...string) {
glog.V(2).Infof("processTenantEndpoint: parentPath:%s hostContainerIDs: %v", parentPath, hostContainerIDs)
// update the proxy for this tenant endpoint
endpointRegistry, err := registry.CreateEndpointRegistry(conn)
if err != nil {
glog.Errorf("Could not get EndpointRegistry. Endpoints not registered: %v", err)
return
}
parts := strings.Split(parentPath, "/")
tenantEndpointID := parts[len(parts)-1]
if ep := c.getMatchingEndpoint(tenantEndpointID); ep != nil {
endpoints := make([]dao.ApplicationEndpoint, len(hostContainerIDs))
for ii, hostContainerID := range hostContainerIDs {
path := fmt.Sprintf("%s/%s", parentPath, hostContainerID)
endpointNode, err := endpointRegistry.GetItem(conn, path)
if err != nil {
glog.Errorf("error getting endpoint node at %s: %v", path, err)
continue
}
endpoints[ii] = endpointNode.ApplicationEndpoint
if ep.port != 0 {
glog.V(2).Infof("overriding ProxyPort with imported port:%v for endpoint: %+v", ep.port, endpointNode)
endpoints[ii].ProxyPort = ep.port
} else {
glog.V(2).Infof("not overriding ProxyPort with imported port:%v for endpoint: %+v", ep.port, endpointNode)
endpoints[ii].ProxyPort = endpoints[ii].ContainerPort
}
}
c.setProxyAddresses(tenantEndpointID, endpoints, ep.virtualAddress, ep.purpose)
}
}
func (f *Facade) RestoreIPs(ctx datastore.Context, svc service.Service) error {
for _, ep := range svc.Endpoints {
if ep.AddressAssignment.IPAddr != "" {
if assign, err := f.FindAssignmentByServiceEndpoint(ctx, svc.ID, ep.Name); err != nil {
glog.Errorf("Could not look up address assignment %s for service %s (%s): %s", ep.Name, svc.Name, svc.ID, err)
return err
} else if assign == nil || !assign.EqualIP(ep.AddressAssignment) {
ip, err := f.getManualAssignment(ctx, svc.PoolID, ep.AddressAssignment.IPAddr, ep.AddressConfig.Port)
if err != nil {
glog.Warningf("Could not assign ip (%s) to endpoint %s for service %s (%s): %s", ep.AddressAssignment.IPAddr, ep.Name, svc.Name, svc.ID, err)
continue
}
assign = &addressassignment.AddressAssignment{
AssignmentType: ip.Type,
HostID: ip.HostID,
PoolID: svc.PoolID,
IPAddr: ip.IP,
Port: ep.AddressConfig.Port,
ServiceID: svc.ID,
EndpointName: ep.Name,
}
if _, err := f.assign(ctx, *assign); err != nil {
glog.Errorf("Could not restore address assignment for %s of service %s at %s:%d: %s", assign.EndpointName, assign.ServiceID, assign.IPAddr, assign.Port, err)
return err
}
glog.Infof("Restored address assignment for endpoint %s of service %s at %s:%d", assign.EndpointName, assign.ServiceID, assign.IPAddr, assign.Port)
} else {
glog.Infof("Endpoint %s for service %s (%s) already assigned; skipping", assign.EndpointName, assign.ServiceID)
}
}
}
return nil
}
// Run() an instance of this container and return it's exec.Command reference and a
// channel that sends the exit code, when the container exits
func (c *Container) run() (*exec.Cmd, chan error) {
// the container name is semi random because containers can get wedged
// in docker and can not be removed until a reboot (or aufs trickery)
containerName := c.Name + "-" + uuid()
exitChan := make(chan error, 1)
args := make([]string, 0)
args = append(args, "run", "-rm", "-name", containerName)
// attach all exported ports
for _, port := range c.Ports {
args = append(args, "-p", fmt.Sprintf("%d:%d", port, port))
}
// attach resources directory to all containers
args = append(args, "-v", resourcesDir()+":"+"/usr/local/serviced/resources")
// attach all exported volumes
for name, volume := range c.Volumes {
hostDir := path.Join(c.VolumesDir(), c.Name, name)
if exists, _ := isDir(hostDir); !exists {
if err := os.MkdirAll(hostDir, 0777); err != nil {
glog.Errorf("could not create %s on host: %s", hostDir, err)
exitChan <- err
return nil, exitChan
}
}
args = append(args, "-v", hostDir+":"+volume)
}
// set the image and command to run
args = append(args, c.Repo+":"+c.Tag, "/bin/sh", "-c", c.Command)
glog.V(1).Infof("Executing docker %s", args)
var cmd *exec.Cmd
tries := 5
var err error
for {
if tries > 0 {
cmd = exec.Command("docker", args...)
if err := cmd.Start(); err != nil {
glog.Errorf("Could not start: %s", c.Name)
c.stop()
c.rm()
time.Sleep(time.Second * 1)
} else {
break
}
} else {
exitChan <- err
return cmd, exitChan
}
tries = -1
}
go func() {
exitChan <- cmd.Wait()
}()
return cmd, exitChan
}
//restUpdateHost updates a host. Request input is host.Host
func restUpdateHost(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
hostID, err := url.QueryUnescape(r.PathParam("hostId"))
if err != nil {
restBadRequest(w, err)
return
}
glog.V(3).Infof("Received update request for %s", hostID)
var payload host.Host
err = r.DecodeJsonPayload(&payload)
if err != nil {
glog.V(1).Infof("Could not decode host payload: %v", err)
restBadRequest(w, err)
return
}
masterClient, err := ctx.getMasterClient()
if err != nil {
glog.Errorf("Unable to add host: %v", err)
restServerError(w, err)
return
}
err = masterClient.UpdateHost(payload)
if err != nil {
glog.Errorf("Unable to update host: %v", err)
restServerError(w, err)
return
}
glog.V(1).Info("Updated host ", hostID)
w.WriteJson(&simpleResponse{"Updated host", hostLinks(hostID)})
}
func restGetAllRunning(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
var services []dao.RunningService
err := client.GetRunningServices(&empty, &services)
if err != nil {
glog.Errorf("Could not get services: %v", err)
restServerError(w, err)
return
}
if services == nil {
glog.V(3).Info("Services was nil, returning empty list instead")
services = []dao.RunningService{}
}
for ii, rsvc := range services {
var svc service.Service
if err := client.GetService(rsvc.ServiceID, &svc); err != nil {
glog.Errorf("Could not get services: %v", err)
restServerError(w, err)
}
fillBuiltinMetrics(&svc)
services[ii].MonitoringProfile = svc.MonitoringProfile
}
services = append(services, getIRS()...)
glog.V(2).Infof("Return %d running services", len(services))
w.WriteJson(&services)
}
// fillBuiltinMetrics adds internal metrics to the monitoring profile
func fillBuiltinMetrics(svc *service.Service) {
if strings.HasPrefix(svc.ID, "isvc-") {
return
}
if svc.MonitoringProfile.MetricConfigs == nil {
builder, err := domain.NewMetricConfigBuilder("/metrics/api/performance/query", "POST")
if err != nil {
glog.Errorf("Could not create builder to add internal metrics: %s", err)
return
}
config, err := builder.Config("metrics", "metrics", "metrics", "-1h")
if err != nil {
glog.Errorf("could not create metric config for internal metrics: %s", err)
}
svc.MonitoringProfile.MetricConfigs = []domain.MetricConfig{*config}
}
index, found := findInternalMetricConfig(svc)
if !found {
glog.Errorf("should have been able to find internal metrics config")
return
}
config := &svc.MonitoringProfile.MetricConfigs[index]
removeInternalMetrics(config)
removeInternalGraphConfigs(svc)
if len(svc.Startup) > 2 {
addInternalMetrics(config)
addInternalGraphConfigs(svc)
}
}
func (z *zkf) CheckRunningVHost(vhostName, serviceID string) error {
rootBasedConnection, err := zzk.GetLocalConnection("/")
if err != nil {
return err
}
vr, err := zkregistry.VHostRegistry(rootBasedConnection)
if err != nil {
glog.Errorf("Error getting vhost registry: %v", err)
return err
}
vhostEphemeralNodes, err := vr.GetVHostKeyChildren(rootBasedConnection, vhostName)
if err != nil {
glog.Errorf("GetVHostKeyChildren failed %v: %v", vhostName, err)
return err
}
if len(vhostEphemeralNodes) > 0 {
if vhost := vhostEphemeralNodes[0]; vhost.ServiceID != serviceID {
err := fmt.Errorf("virtual host %s is already running under service %s", vhostName, vhost.ServiceID)
return err
}
}
return nil
}
// validate the provided service
func (f *Facade) validateService(ctx datastore.Context, serviceId string, autoLaunch bool) error {
//TODO: create map of IPs to ports and ensure that an IP does not have > 1 process listening on the same port
visitor := func(svc *service.Service) error {
// validate the service is ready to start
err := f.validateServicesForStarting(ctx, svc)
if err != nil {
glog.Errorf("services failed validation for starting")
return err
}
for _, ep := range svc.GetServiceVHosts() {
for _, vh := range ep.VHosts {
//check that vhosts aren't already started elsewhere
if err := zkAPI(f).CheckRunningVHost(vh, svc.ID); err != nil {
return err
}
}
}
return nil
}
// traverse all the services
if err := f.walkServices(ctx, serviceId, autoLaunch, visitor); err != nil {
glog.Errorf("unable to walk services for service %s", serviceId)
return err
}
return nil
}
func (this *ControlPlaneDao) GetRunningServicesForService(serviceID string, services *[]dao.RunningService) error {
// we initialize the data container to something here in case it has not been initialized yet
*services = make([]dao.RunningService, 0)
poolID, err := this.facade.GetPoolForService(datastore.Get(), serviceID)
if err != nil {
glog.Errorf("Unable to get service %v: %v", serviceID, err)
return err
}
poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID))
if err != nil {
glog.Errorf("Error in getting a connection based on pool %v: %v", poolID, err)
return err
}
svcs, err := zkservice.LoadRunningServicesByService(poolBasedConn, serviceID)
if err != nil {
glog.Errorf("LoadRunningServicesByService failed (conn: %+v serviceID: %v): %v", poolBasedConn, serviceID, err)
return err
}
for _, svc := range svcs {
*services = append(*services, svc)
}
return nil
}
func (svc *IService) remove(notify chan<- int) {
defer close(notify)
ctr, err := docker.FindContainer(svc.name())
if err == docker.ErrNoSuchContainer {
return
} else if err != nil {
glog.Errorf("Could not get isvc container %s", svc.Name)
return
}
// report the log output
if output, err := exec.Command("docker", "logs", "--tail", "1000", ctr.ID).CombinedOutput(); err != nil {
glog.Warningf("Could not get logs for container %s", ctr.Name)
} else {
glog.V(1).Infof("Exited isvc %s:\n %s", svc.Name, string(output))
}
// kill the container if it is running
if ctr.IsRunning() {
glog.Warningf("isvc %s is still running; killing", svc.Name)
ctr.Kill()
}
// get the exit code
rc, _ := ctr.Wait(time.Second)
defer func() { notify <- rc }()
// delete the container
if err := ctr.Delete(true); err != nil {
glog.Errorf("Could not remove isvc %s: %s", ctr.Name, err)
}
}
func (c *Controller) kickOffHealthChecks(healthExit chan struct{}) {
client, err := node.NewLBClient(c.options.ServicedEndpoint)
if err != nil {
glog.Errorf("Could not create a client to endpoint: %s, %s", c.options.ServicedEndpoint, err)
return
}
defer client.Close()
var healthChecks map[string]domain.HealthCheck
instanceID, err := strconv.Atoi(c.options.Service.InstanceID)
if err != nil {
glog.Errorf("Invalid instance from instanceID:%s", c.options.Service.InstanceID)
return
}
err = client.GetHealthCheck(node.HealthCheckRequest{
c.options.Service.ID, instanceID}, &healthChecks)
if err != nil {
glog.Errorf("Error getting health checks: %s", err)
return
}
for key, mapping := range healthChecks {
glog.Infof("Kicking off health check %s.", key)
glog.Infof("Setting up health check: %s", mapping.Script)
timeout := mapping.Timeout
if timeout == 0 {
timeout = time.Second * 30
}
go c.handleHealthCheck(key, mapping.Script, mapping.Interval, timeout, healthExit)
}
return
}
// NewMetricConfigBuilder creates a factory to create MetricConfig instances.
func NewMetricConfigBuilder(RequestURI, Method string) (*MetricBuilder, error) {
//strip leading '/' it's added back below
requestURI := RequestURI
if len(RequestURI) > 0 && RequestURI[0] == '/' {
requestURI = RequestURI[1:]
}
//use url.Parse to ensure proper RequestURI. 'http://localhost' is removed when Config is built
url, err := url.Parse("http://localhost/" + requestURI)
if err != nil {
glog.Errorf("Invalid Url: RequestURI=%s, method=%s, err=%+v", RequestURI, Method, err)
return nil, err
}
method := strings.ToUpper(Method)
switch method {
case "GET":
case "PUT":
case "POST":
default:
glog.Errorf("Invalid http method: RequestURI=%s, method=%s", RequestURI, Method)
return nil, errors.New("invalid method")
}
return &MetricBuilder{
url: url,
method: Method,
metrics: make([]MetricMetricBuilder, 0),
}, nil
}
/*
registerIPs does the actual work of determining the IPs on the host. Parameters are the hostId for this host
and the function used to send the found IPs
*/
func registerIPs(hostId string, sendFn SendHostIPs) error {
interfaces, err := net.Interfaces()
if err != nil {
glog.Error("Problem reading interfaces: ", err)
return err
}
hostIPResources := make([]dao.HostIPResource, 0, len(interfaces))
for _, iface := range interfaces {
addrs, err := iface.Addrs()
if err != nil {
glog.Errorf("Problem reading address for interface %s: %s", iface.Name, err)
return err
}
for _, addr := range addrs {
//send address to Master
hostIp := dao.HostIPResource{}
hostIp.IPAddress = addr.String()
hostIp.InterfaceName = iface.Name
hostIPResources = append(hostIPResources, hostIp)
}
}
var unused int
glog.V(4).Infof("Agent registering IPs %v", hostIPResources)
hostIps := dao.HostIPs{}
hostIps.HostId = hostId
hostIps.IPs = hostIPResources
if err := sendFn(hostIps, &unused); err != nil {
glog.Errorf("Error registering IPs %v", err)
}
return nil
}
func RestGetService(w *rest.ResponseWriter, r *rest.Request, client *serviced.ControlClient) {
var allServices []*dao.Service
if err := client.GetServices(&empty, &allServices); err != nil {
glog.Errorf("Could not get services: %v", err)
RestServerError(w)
return
}
sid, err := url.QueryUnescape(r.PathParam("serviceId"))
if err != nil {
RestBadRequest(w)
return
}
for _, service := range allServices {
if service.Id == sid {
w.WriteJson(&service)
return
}
}
glog.Errorf("No such service [%v]", sid)
RestServerError(w)
}
// restRemovePoolVirtualIP deletes virtualip
func restRemovePoolVirtualIP(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
ip, err := url.QueryUnescape(r.PathParam("ip"))
if err != nil {
glog.Errorf("Could not get virtual ip (%v): %v", ip, err)
restBadRequest(w, err)
return
}
poolId, err := url.QueryUnescape(r.PathParam("poolId"))
if err != nil {
glog.Errorf("Could not get virtual ip poolId (%v): %v", poolId, err)
restBadRequest(w, err)
return
}
glog.V(0).Infof("Remove virtual ip=%v (in pool %v)", ip, poolId)
client, err := ctx.getMasterClient()
if err != nil {
restServerError(w, err)
return
}
request := pool.VirtualIP{PoolID: poolId, IP: ip}
if err := client.RemoveVirtualIP(request); err != nil {
glog.Errorf("Failed to remove virtual IP(%+v): %v", request, err)
restServerError(w, err)
return
}
restSuccess(w)
}
// Retrieve service container port info.
func (ss *ServiceState) GetHostEndpointInfo(applicationRegex *regexp.Regexp) (hostPort, containerPort uint16, protocol string, match bool) {
for _, ep := range ss.Endpoints {
if ep.Purpose == "export" {
if applicationRegex.MatchString(ep.Application) {
if ep.PortTemplate != "" {
port, err := ss.evalPortTemplate(ep.PortTemplate)
if err != nil {
glog.Errorf("%+v", err)
break
}
ep.PortNumber = uint16(port)
}
portS := fmt.Sprintf("%d/%s", ep.PortNumber, strings.ToLower(ep.Protocol))
external := ss.PortMapping[portS]
if len(external) == 0 {
glog.Warningf("Found match for %s:%s, but no portmapping is available", applicationRegex, portS)
break
}
extPort, err := strconv.ParseUint(external[0].HostPort, 10, 16)
if err != nil {
glog.Errorf("Portmap parsing failed for %s:%s %v", applicationRegex, portS, err)
break
}
return uint16(extPort), ep.PortNumber, ep.Protocol, true
}
}
}
return 0, 0, "", false
}
func (ed *elasticDriver) getHealth() (map[string]interface{}, error) {
health := make(map[string]interface{})
healthURL := fmt.Sprintf("%v/_cluster/health", ed.elasticURL())
resp, err := http.Get(healthURL)
if err != nil {
return health, err
}
if resp.StatusCode != 200 {
return health, fmt.Errorf("http status: %v", resp.StatusCode)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
glog.Errorf("error reading elastic health: %v", err)
return health, err
}
if err := json.Unmarshal(body, &health); err != nil {
glog.Errorf("error unmarshalling elastic health: %v; err: %v", string(body), err)
return health, err
}
glog.V(4).Infof("Elastic Health: %v; err: %v", string(body), err)
return health, nil
}
func (viface *vif) redirectCommand(from, to, protocol string) error {
glog.Infof("Trying to set up redirect %s:%s->:%s %s", viface.hostname, from, to, protocol)
for _, chain := range []string{"OUTPUT", "PREROUTING"} {
command := []string{
"iptables",
"-t", "nat",
"-A", chain,
"-d", viface.ip,
"-p", protocol,
"--dport", from,
"-j", "REDIRECT",
"--to-ports", to,
}
c := exec.Command(command[0], command[1:]...)
c.Stdout = os.Stdout
c.Stderr = os.Stdout
if err := c.Run(); err != nil {
glog.Errorf("Unable to set up redirect %s:%s->:%s %s command:%+v", viface.hostname, from, to, protocol, command)
return err
}
}
glog.Infof("AddToEtcHosts(%s, %s)", viface.hostname, viface.ip)
err := node.AddToEtcHosts(viface.hostname, viface.ip)
if err != nil {
glog.Errorf("Unable to add %s %s to /etc/hosts", viface.ip, viface.hostname)
return err
}
return nil
}
func restGetService(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
sid, err := url.QueryUnescape(r.PathParam("serviceId"))
if err != nil {
restBadRequest(w, err)
return
}
if strings.Contains(sid, "isvc-") {
w.WriteJson(isvcs.ISVCSMap[sid])
return
}
svc := service.Service{}
if err := client.GetService(sid, &svc); err != nil {
glog.Errorf("Could not get service %v: %v", sid, err)
restServerError(w, err)
return
}
if svc.ID == sid {
fillBuiltinMetrics(&svc)
w.WriteJson(&svc)
return
}
glog.Errorf("No such service [%v]", sid)
restServerError(w, err)
}
func (dfs *DistributedFilesystem) importImages(dirpath string, images []imagemeta, tenants map[string]struct{}) error {
for _, metadata := range images {
filename := filepath.Join(dirpath, metadata.Filename)
// Make sure all images that refer to a local registry are named with the local registry
tags := make([]string, len(metadata.Tags))
for i, tag := range metadata.Tags {
imageID, err := commons.ParseImageID(tag)
if err != nil {
glog.Errorf("Could not parse %s: %s", tag, err)
return err
}
if _, ok := tenants[imageID.User]; ok {
imageID.Host, imageID.Port = dfs.dockerHost, dfs.dockerPort
}
tags[i] = imageID.String()
}
if err := loadImage(filename, metadata.UUID, tags); err != nil {
glog.Errorf("Error loading %s (%s): %s", filename, metadata.UUID, err)
return err
}
}
return nil
}
func (svc *IService) start() (<-chan int, error) {
ctr, err := svc.attach()
if err != nil {
svc.setStoppedHealthStatus(fmt.Errorf("could not start service: %s", err))
return nil, err
}
// destroy the container when it dies
notify := make(chan int, 1)
ctr.OnEvent(docker.Die, func(cid string) { svc.remove(notify) })
// start the container
if err := ctr.Start(10 * time.Second); err != nil && err != docker.ErrAlreadyStarted {
svc.setStoppedHealthStatus(fmt.Errorf("could not start service: %s", err))
return nil, err
}
// perform an initial healthcheck to verify that the service started successfully
select {
case err := <-svc.startupHealthcheck():
if err != nil {
glog.Errorf("Healthcheck for %s failed: %s", svc.Name, err)
svc.stop()
return nil, err
}
svc.startTime = time.Now()
case rc := <-notify:
glog.Errorf("isvc %s exited on startup, rc=%d", svc.Name, rc)
svc.setStoppedHealthStatus(ExitError(rc))
return nil, ExitError(rc)
}
return notify, nil
}
func (this *ControlPlaneDao) GetRunningService(request dao.ServiceStateRequest, running *dao.RunningService) error {
glog.V(3).Infof("ControlPlaneDao.GetRunningService: request=%v", request)
*running = dao.RunningService{}
serviceID := request.ServiceID
poolID, err := this.facade.GetPoolForService(datastore.Get(), serviceID)
if err != nil {
glog.Errorf("Unable to get service %v: %v", serviceID, err)
return err
}
poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID))
if err != nil {
glog.Errorf("Error in getting a connection based on pool %v: %v", poolID, err)
return err
}
if thisRunning, err := zkservice.LoadRunningService(poolBasedConn, request.ServiceID, request.ServiceStateID); err != nil {
glog.Errorf("zkservice.LoadRunningService failed (conn: %+v serviceID: %v): %v", poolBasedConn, request.ServiceID, err)
return err
} else {
if thisRunning != nil {
*running = *thisRunning
}
}
return nil
}
// Snapshots returns the current snapshots on the volume (sorted by date)
func (c *BtrfsConn) Snapshots() ([]string, error) {
c.Lock()
defer c.Unlock()
glog.V(2).Infof("listing snapshots of volume:%v and c.name:%s ", c.root, c.name)
output, err := runcmd(c.sudoer, "subvolume", "list", "-s", c.root)
if err != nil {
glog.Errorf("Could not list subvolumes of %s: %s", c.root, err)
return nil, err
}
var files []os.FileInfo
for _, line := range strings.Split(string(output), "\n") {
glog.V(2).Infof("line: %s", line)
if parts := strings.Split(line, "path"); len(parts) == 2 {
label := strings.TrimSpace(parts[1])
label = strings.TrimPrefix(label, "volumes/")
glog.V(2).Infof("looking for tenant:%s in label:'%s'", c.name, label)
if strings.HasPrefix(label, c.name+"_") {
file, err := os.Stat(filepath.Join(c.root, label))
if err != nil {
glog.Errorf("Could not stat snapshot %s: %s", label, err)
return nil, err
}
files = append(files, file)
glog.V(2).Infof("found snapshot:%s", label)
}
}
}
return volume.FileInfoSlice(files).Labels(), nil
}
func (this *ControlPlaneDao) GetRunningServicesForHost(hostID string, services *[]dao.RunningService) error {
// we initialize the data container to something here in case it has not been initialized yet
*services = make([]dao.RunningService, 0)
myHost, err := this.facade.GetHost(datastore.Get(), hostID)
if err != nil {
glog.Errorf("Unable to get host %v: %v", hostID, err)
return err
} else if myHost == nil {
return nil
}
poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(myHost.PoolID))
if err != nil {
glog.Errorf("Error in getting a connection based on pool %v: %v", myHost.PoolID, err)
return err
}
*services, err = zkservice.LoadRunningServicesByHost(poolBasedConn, hostID)
if err != nil {
glog.Errorf("zkservice.LoadRunningServicesByHost (conn: %+v host: %v) failed: %v", poolBasedConn, hostID, err)
return err
}
return nil
}
// Spawn starts the remote Synchronizer based on nodeID
func (l *Synchronizer) Spawn(shutdown <-chan interface{}, nodeID string) {
// Start dependent listeners
var (
wg sync.WaitGroup
_shutdown = make(chan interface{})
stopping = make(chan interface{})
)
defer func() {
close(_shutdown)
wg.Wait()
}()
var id string
var wait <-chan time.Time
for {
node := l.Allocate()
event, err := l.conn.GetW(l.GetPath(nodeID), node)
if err == client.ErrNoNode && id != "" {
if err := l.Delete(id); err != nil {
glog.Errorf("Could not delete node at %s: %s", l.GetPath(nodeID), err)
wait = time.After(DefaultRetryTime)
} else {
return
}
} else if err != nil {
glog.Errorf("Could not get node at %s: %s", l.GetPath(nodeID), err)
return
}
if key, err := l.AddUpdate(id, node); err == ErrInvalidType {
glog.Errorf("Invalid type detected")
return
} else if err != nil {
glog.Errorf("Could not update node at %s: %s", l.GetPath(nodeID), err)
wait = time.After(DefaultRetryTime)
} else if id == "" {
wg.Add(1)
go func() {
defer wg.Done()
if len(l.lfuncs) == 0 {
return
}
defer close(stopping)
l.startListeners(_shutdown, nodeID)
}()
id = key
}
select {
case <-event:
case <-wait:
case <-stopping:
return
case <-shutdown:
return
}
}
}
func (f *Facade) getAutoAssignment(ctx datastore.Context, poolID string, ports ...uint16) (ipinfo, error) {
pool, err := f.GetResourcePool(ctx, poolID)
if err != nil {
glog.Errorf("Error while looking up pool %s: %s", poolID, err)
return ipinfo{}, err
}
ignoreips := make(map[string]struct{})
for _, port := range ports {
// Get all of the address assignments for port
assignments, err := f.GetServiceAddressAssignmentsByPort(ctx, port)
if err != nil {
glog.Errorf("Error while looking up address assignments for port %d: %s", port, err)
return ipinfo{}, err
}
// Find out all of the host ips that cannot be used
for _, assignment := range assignments {
ignoreips[assignment.IPAddr] = struct{}{}
}
}
// Filter virtual ips
var ips []ipinfo
for _, vip := range pool.VirtualIPs {
if _, ok := ignoreips[vip.IP]; !ok {
ips = append(ips, ipinfo{vip.IP, commons.VIRTUAL, ""})
}
}
hosts, err := f.FindHostsInPool(ctx, poolID)
if err != nil {
glog.Errorf("Error while looking up hosts in pool %s: %s", poolID, err)
return ipinfo{}, err
}
var resources []host.HostIPResource
for _, host := range hosts {
if host.IPs != nil {
resources = append(resources, host.IPs...)
}
}
// Filter static ips
for _, hostIP := range resources {
if _, ok := ignoreips[hostIP.IPAddress]; !ok {
ips = append(ips, ipinfo{hostIP.IPAddress, commons.STATIC, hostIP.HostID})
}
}
// Pick an ip
total := len(ips)
if total == 0 {
err := fmt.Errorf("No IPs are available to be assigned")
glog.Errorf("Error acquiring IP assignment: %s", err)
return ipinfo{}, err
}
rand.Seed(time.Now().UTC().UnixNano())
return ips[rand.Intn(total)], nil
}