Example #1
0
func (this *ControlPlaneDao) GetServiceLogs(serviceID string, logs *string) error {
	glog.V(3).Info("ControlPlaneDao.GetServiceLogs serviceID=", serviceID)
	var serviceStates []servicestate.ServiceState
	if err := this.GetServiceStates(serviceID, &serviceStates); err != nil {
		glog.Errorf("ControlPlaneDao.GetServiceLogs failed: %v", err)
		return err
	}

	if len(serviceStates) == 0 {
		glog.V(1).Info("Unable to find any running services for service:", serviceID)
		return nil
	}

	serviceState := serviceStates[0]
	endpoint := fmt.Sprintf("%s:%d", serviceState.HostIP, this.rpcPort)
	agentClient, err := agent.NewClient(endpoint)
	if err != nil {
		glog.Errorf("could not create client to %s", endpoint)
		return err
	}

	defer agentClient.Close()
	if mylogs, err := agentClient.GetDockerLogs(serviceState.DockerID); err != nil {
		glog.Errorf("could not get docker logs from agent client: %s", err)
		return err
	} else {
		*logs = mylogs
	}
	return nil
}
Example #2
0
func (f *Facade) RemoveService(ctx datastore.Context, id string) error {
	store := f.serviceStore

	return f.walkServices(ctx, id, true, func(svc *service.Service) error {
		// remove all address assignments
		for _, endpoint := range svc.Endpoints {
			if assignment, err := f.FindAssignmentByServiceEndpoint(ctx, svc.ID, endpoint.Name); err != nil {
				glog.Errorf("Could not find address assignment %s for service %s (%s): %s", endpoint.Name, svc.Name, svc.ID, err)
				return err
			} else if assignment != nil {
				if err := f.RemoveAddressAssignment(ctx, assignment.ID); err != nil {
					glog.Errorf("Could not remove address assignment %s from service %s (%s): %s", endpoint.Name, svc.Name, svc.ID, err)
					return err
				}
			}
			endpoint.RemoveAssignment()
		}

		if err := zkAPI(f).RemoveService(svc); err != nil {
			glog.Errorf("Could not remove service %s (%s) from zookeeper: %s", svc.Name, svc.ID, err)
			return err
		}

		if err := store.Delete(ctx, svc.ID); err != nil {
			glog.Errorf("Error while removing service %s (%s): %s", svc.Name, svc.ID, err)
			return err
		}

		return nil
	})
}
Example #3
0
//ValidEntity validates Host fields
func (h *Host) ValidEntity() error {
	glog.V(4).Info("Validating host")

	//if err := validation.ValidHostID(entity.ID); err != nil {
	//	return fmt.Errorf("invalid hostid:'%s' for host Name:'%s' IP:%s", entity.ID, entity.Name, entity.IPAddr)
	//}

	trimmedID := strings.TrimSpace(h.ID)
	violations := validation.NewValidationError()
	violations.Add(validation.NotEmpty("Host.ID", h.ID))
	violations.Add(validation.ValidHostID(h.ID))
	violations.Add(validation.StringsEqual(h.ID, trimmedID, "leading and trailing spaces not allowed for host id"))
	violations.Add(validation.ValidPort(h.RPCPort))
	violations.Add(validation.NotEmpty("Host.PoolID", h.PoolID))
	violations.Add(validation.IsIP(h.IPAddr))

	//TODO: what should we be validating here? It doesn't seem to work for
	glog.V(4).Infof("Validating IPAddr %v for host %s", h.IPAddr, h.ID)
	ipAddr, err := net.ResolveIPAddr("ip4", h.IPAddr)

	if err != nil {
		glog.Errorf("Could not resolve: %s to an ip4 address: %v", h.IPAddr, err)
		violations.Add(err)
	} else if ipAddr.IP.IsLoopback() {
		glog.Errorf("Can not use %s as host address because it is a loopback address", h.IPAddr)
		violations.Add(errors.New("host ip can not be a loopback address"))

	}

	if len(violations.Errors) > 0 {
		return violations
	}
	return nil
}
Example #4
0
// getServiceState gets the service state for a serviceID matching the instance ID specified
func getServiceState(conn coordclient.Connection, serviceID, instanceIDStr string) (*servicestate.ServiceState, error) {
	tmpID, err := strconv.Atoi(instanceIDStr)
	if err != nil {
		glog.Errorf("Unable to interpret InstanceID: %s", instanceIDStr)
		return nil, fmt.Errorf("endpoint.go getServiceState failed: %v", err)
	}
	instanceID := int(tmpID)

	for {
		serviceStates, err := getServiceStates(conn, serviceID)
		if err != nil {
			glog.Errorf("Unable to retrieve running service (%s) states: %v", serviceID, err)
			return nil, fmt.Errorf("endpoint.go getServiceState zzk.GetServiceStates failed: %v", err)
		}

		for ii, ss := range serviceStates {
			if ss.InstanceID == instanceID && ss.PrivateIP != "" {
				return &serviceStates[ii], nil
			}
		}

		glog.V(2).Infof("Polling to retrieve service state instanceID:%d with valid PrivateIP", instanceID)
		time.Sleep(1 * time.Second)
	}

	return nil, fmt.Errorf("unable to retrieve service state")
}
Example #5
0
// processTenantEndpoint updates the addresses for an imported endpoint
func (c *Controller) processTenantEndpoint(conn coordclient.Connection, parentPath string, hostContainerIDs ...string) {
	glog.V(2).Infof("processTenantEndpoint: parentPath:%s hostContainerIDs: %v", parentPath, hostContainerIDs)

	// update the proxy for this tenant endpoint
	endpointRegistry, err := registry.CreateEndpointRegistry(conn)
	if err != nil {
		glog.Errorf("Could not get EndpointRegistry. Endpoints not registered: %v", err)
		return
	}

	parts := strings.Split(parentPath, "/")
	tenantEndpointID := parts[len(parts)-1]

	if ep := c.getMatchingEndpoint(tenantEndpointID); ep != nil {
		endpoints := make([]dao.ApplicationEndpoint, len(hostContainerIDs))
		for ii, hostContainerID := range hostContainerIDs {
			path := fmt.Sprintf("%s/%s", parentPath, hostContainerID)
			endpointNode, err := endpointRegistry.GetItem(conn, path)
			if err != nil {
				glog.Errorf("error getting endpoint node at %s: %v", path, err)
				continue
			}
			endpoints[ii] = endpointNode.ApplicationEndpoint
			if ep.port != 0 {
				glog.V(2).Infof("overriding ProxyPort with imported port:%v for endpoint: %+v", ep.port, endpointNode)
				endpoints[ii].ProxyPort = ep.port
			} else {
				glog.V(2).Infof("not overriding ProxyPort with imported port:%v for endpoint: %+v", ep.port, endpointNode)
				endpoints[ii].ProxyPort = endpoints[ii].ContainerPort
			}
		}
		c.setProxyAddresses(tenantEndpointID, endpoints, ep.virtualAddress, ep.purpose)
	}
}
Example #6
0
func (f *Facade) RestoreIPs(ctx datastore.Context, svc service.Service) error {
	for _, ep := range svc.Endpoints {
		if ep.AddressAssignment.IPAddr != "" {
			if assign, err := f.FindAssignmentByServiceEndpoint(ctx, svc.ID, ep.Name); err != nil {
				glog.Errorf("Could not look up address assignment %s for service %s (%s): %s", ep.Name, svc.Name, svc.ID, err)
				return err
			} else if assign == nil || !assign.EqualIP(ep.AddressAssignment) {
				ip, err := f.getManualAssignment(ctx, svc.PoolID, ep.AddressAssignment.IPAddr, ep.AddressConfig.Port)
				if err != nil {
					glog.Warningf("Could not assign ip (%s) to endpoint %s for service %s (%s): %s", ep.AddressAssignment.IPAddr, ep.Name, svc.Name, svc.ID, err)
					continue
				}

				assign = &addressassignment.AddressAssignment{
					AssignmentType: ip.Type,
					HostID:         ip.HostID,
					PoolID:         svc.PoolID,
					IPAddr:         ip.IP,
					Port:           ep.AddressConfig.Port,
					ServiceID:      svc.ID,
					EndpointName:   ep.Name,
				}
				if _, err := f.assign(ctx, *assign); err != nil {
					glog.Errorf("Could not restore address assignment for %s of service %s at %s:%d: %s", assign.EndpointName, assign.ServiceID, assign.IPAddr, assign.Port, err)
					return err
				}
				glog.Infof("Restored address assignment for endpoint %s of service %s at %s:%d", assign.EndpointName, assign.ServiceID, assign.IPAddr, assign.Port)
			} else {
				glog.Infof("Endpoint %s for service %s (%s) already assigned; skipping", assign.EndpointName, assign.ServiceID)
			}
		}
	}
	return nil
}
Example #7
0
// Run() an instance of this container and return it's exec.Command reference and a
// channel that sends the exit code, when the container exits
func (c *Container) run() (*exec.Cmd, chan error) {

	// the container name is semi random because containers can get wedged
	// in docker and can not be removed until a reboot (or aufs trickery)
	containerName := c.Name + "-" + uuid()

	exitChan := make(chan error, 1)
	args := make([]string, 0)
	args = append(args, "run", "-rm", "-name", containerName)

	// attach all exported ports
	for _, port := range c.Ports {
		args = append(args, "-p", fmt.Sprintf("%d:%d", port, port))
	}

	// attach resources directory to all containers
	args = append(args, "-v", resourcesDir()+":"+"/usr/local/serviced/resources")

	// attach all exported volumes
	for name, volume := range c.Volumes {
		hostDir := path.Join(c.VolumesDir(), c.Name, name)
		if exists, _ := isDir(hostDir); !exists {
			if err := os.MkdirAll(hostDir, 0777); err != nil {
				glog.Errorf("could not create %s on host: %s", hostDir, err)
				exitChan <- err
				return nil, exitChan
			}
		}
		args = append(args, "-v", hostDir+":"+volume)
	}

	// set the image and command to run
	args = append(args, c.Repo+":"+c.Tag, "/bin/sh", "-c", c.Command)

	glog.V(1).Infof("Executing docker %s", args)
	var cmd *exec.Cmd
	tries := 5
	var err error
	for {
		if tries > 0 {
			cmd = exec.Command("docker", args...)
			if err := cmd.Start(); err != nil {
				glog.Errorf("Could not start: %s", c.Name)
				c.stop()
				c.rm()
				time.Sleep(time.Second * 1)
			} else {
				break
			}
		} else {
			exitChan <- err
			return cmd, exitChan
		}
		tries = -1
	}
	go func() {
		exitChan <- cmd.Wait()
	}()
	return cmd, exitChan
}
Example #8
0
//restUpdateHost updates a host. Request input is host.Host
func restUpdateHost(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	hostID, err := url.QueryUnescape(r.PathParam("hostId"))
	if err != nil {
		restBadRequest(w, err)
		return
	}
	glog.V(3).Infof("Received update request for %s", hostID)
	var payload host.Host
	err = r.DecodeJsonPayload(&payload)
	if err != nil {
		glog.V(1).Infof("Could not decode host payload: %v", err)
		restBadRequest(w, err)
		return
	}

	masterClient, err := ctx.getMasterClient()
	if err != nil {
		glog.Errorf("Unable to add host: %v", err)
		restServerError(w, err)
		return
	}
	err = masterClient.UpdateHost(payload)
	if err != nil {
		glog.Errorf("Unable to update host: %v", err)
		restServerError(w, err)
		return
	}
	glog.V(1).Info("Updated host ", hostID)
	w.WriteJson(&simpleResponse{"Updated host", hostLinks(hostID)})
}
Example #9
0
func restGetAllRunning(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
	var services []dao.RunningService
	err := client.GetRunningServices(&empty, &services)
	if err != nil {
		glog.Errorf("Could not get services: %v", err)
		restServerError(w, err)
		return
	}
	if services == nil {
		glog.V(3).Info("Services was nil, returning empty list instead")
		services = []dao.RunningService{}
	}

	for ii, rsvc := range services {
		var svc service.Service
		if err := client.GetService(rsvc.ServiceID, &svc); err != nil {
			glog.Errorf("Could not get services: %v", err)
			restServerError(w, err)
		}
		fillBuiltinMetrics(&svc)
		services[ii].MonitoringProfile = svc.MonitoringProfile
	}

	services = append(services, getIRS()...)
	glog.V(2).Infof("Return %d running services", len(services))
	w.WriteJson(&services)
}
// fillBuiltinMetrics adds internal metrics to the monitoring profile
func fillBuiltinMetrics(svc *service.Service) {
	if strings.HasPrefix(svc.ID, "isvc-") {
		return
	}

	if svc.MonitoringProfile.MetricConfigs == nil {
		builder, err := domain.NewMetricConfigBuilder("/metrics/api/performance/query", "POST")
		if err != nil {
			glog.Errorf("Could not create builder to add internal metrics: %s", err)
			return
		}
		config, err := builder.Config("metrics", "metrics", "metrics", "-1h")
		if err != nil {
			glog.Errorf("could not create metric config for internal metrics: %s", err)
		}
		svc.MonitoringProfile.MetricConfigs = []domain.MetricConfig{*config}
	}
	index, found := findInternalMetricConfig(svc)
	if !found {
		glog.Errorf("should have been able to find internal metrics config")
		return
	}
	config := &svc.MonitoringProfile.MetricConfigs[index]
	removeInternalMetrics(config)
	removeInternalGraphConfigs(svc)

	if len(svc.Startup) > 2 {
		addInternalMetrics(config)
		addInternalGraphConfigs(svc)
	}
}
Example #11
0
func (z *zkf) CheckRunningVHost(vhostName, serviceID string) error {
	rootBasedConnection, err := zzk.GetLocalConnection("/")
	if err != nil {
		return err
	}

	vr, err := zkregistry.VHostRegistry(rootBasedConnection)
	if err != nil {
		glog.Errorf("Error getting vhost registry: %v", err)
		return err
	}

	vhostEphemeralNodes, err := vr.GetVHostKeyChildren(rootBasedConnection, vhostName)
	if err != nil {
		glog.Errorf("GetVHostKeyChildren failed %v: %v", vhostName, err)
		return err
	}

	if len(vhostEphemeralNodes) > 0 {
		if vhost := vhostEphemeralNodes[0]; vhost.ServiceID != serviceID {
			err := fmt.Errorf("virtual host %s is already running under service %s", vhostName, vhost.ServiceID)
			return err
		}
	}

	return nil
}
Example #12
0
// validate the provided service
func (f *Facade) validateService(ctx datastore.Context, serviceId string, autoLaunch bool) error {
	//TODO: create map of IPs to ports and ensure that an IP does not have > 1 process listening on the same port
	visitor := func(svc *service.Service) error {
		// validate the service is ready to start
		err := f.validateServicesForStarting(ctx, svc)
		if err != nil {
			glog.Errorf("services failed validation for starting")
			return err
		}
		for _, ep := range svc.GetServiceVHosts() {
			for _, vh := range ep.VHosts {
				//check that vhosts aren't already started elsewhere
				if err := zkAPI(f).CheckRunningVHost(vh, svc.ID); err != nil {
					return err
				}
			}
		}
		return nil
	}

	// traverse all the services
	if err := f.walkServices(ctx, serviceId, autoLaunch, visitor); err != nil {
		glog.Errorf("unable to walk services for service %s", serviceId)
		return err
	}

	return nil
}
Example #13
0
func (this *ControlPlaneDao) GetRunningServicesForService(serviceID string, services *[]dao.RunningService) error {
	// we initialize the data container to something here in case it has not been initialized yet
	*services = make([]dao.RunningService, 0)

	poolID, err := this.facade.GetPoolForService(datastore.Get(), serviceID)
	if err != nil {
		glog.Errorf("Unable to get service %v: %v", serviceID, err)
		return err
	}

	poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID))
	if err != nil {
		glog.Errorf("Error in getting a connection based on pool %v: %v", poolID, err)
		return err
	}

	svcs, err := zkservice.LoadRunningServicesByService(poolBasedConn, serviceID)
	if err != nil {
		glog.Errorf("LoadRunningServicesByService failed (conn: %+v serviceID: %v): %v", poolBasedConn, serviceID, err)
		return err
	}

	for _, svc := range svcs {
		*services = append(*services, svc)
	}

	return nil
}
Example #14
0
func (svc *IService) remove(notify chan<- int) {
	defer close(notify)
	ctr, err := docker.FindContainer(svc.name())
	if err == docker.ErrNoSuchContainer {
		return
	} else if err != nil {
		glog.Errorf("Could not get isvc container %s", svc.Name)
		return
	}

	// report the log output
	if output, err := exec.Command("docker", "logs", "--tail", "1000", ctr.ID).CombinedOutput(); err != nil {
		glog.Warningf("Could not get logs for container %s", ctr.Name)
	} else {
		glog.V(1).Infof("Exited isvc %s:\n %s", svc.Name, string(output))
	}

	// kill the container if it is running
	if ctr.IsRunning() {
		glog.Warningf("isvc %s is still running; killing", svc.Name)
		ctr.Kill()
	}

	// get the exit code
	rc, _ := ctr.Wait(time.Second)
	defer func() { notify <- rc }()

	// delete the container
	if err := ctr.Delete(true); err != nil {
		glog.Errorf("Could not remove isvc %s: %s", ctr.Name, err)
	}
}
Example #15
0
func (c *Controller) kickOffHealthChecks(healthExit chan struct{}) {
	client, err := node.NewLBClient(c.options.ServicedEndpoint)
	if err != nil {
		glog.Errorf("Could not create a client to endpoint: %s, %s", c.options.ServicedEndpoint, err)
		return
	}
	defer client.Close()
	var healthChecks map[string]domain.HealthCheck

	instanceID, err := strconv.Atoi(c.options.Service.InstanceID)
	if err != nil {
		glog.Errorf("Invalid instance from instanceID:%s", c.options.Service.InstanceID)
		return
	}
	err = client.GetHealthCheck(node.HealthCheckRequest{
		c.options.Service.ID, instanceID}, &healthChecks)
	if err != nil {
		glog.Errorf("Error getting health checks: %s", err)
		return
	}
	for key, mapping := range healthChecks {
		glog.Infof("Kicking off health check %s.", key)
		glog.Infof("Setting up health check: %s", mapping.Script)
		timeout := mapping.Timeout
		if timeout == 0 {
			timeout = time.Second * 30
		}
		go c.handleHealthCheck(key, mapping.Script, mapping.Interval, timeout, healthExit)
	}
	return
}
Example #16
0
// NewMetricConfigBuilder creates a factory to create MetricConfig instances.
func NewMetricConfigBuilder(RequestURI, Method string) (*MetricBuilder, error) {
	//strip leading '/' it's added back below
	requestURI := RequestURI
	if len(RequestURI) > 0 && RequestURI[0] == '/' {
		requestURI = RequestURI[1:]
	}

	//use url.Parse to ensure proper RequestURI. 'http://localhost' is removed when Config is built
	url, err := url.Parse("http://localhost/" + requestURI)
	if err != nil {
		glog.Errorf("Invalid Url: RequestURI=%s, method=%s, err=%+v", RequestURI, Method, err)
		return nil, err
	}

	method := strings.ToUpper(Method)
	switch method {
	case "GET":
	case "PUT":
	case "POST":
	default:
		glog.Errorf("Invalid http method: RequestURI=%s, method=%s", RequestURI, Method)
		return nil, errors.New("invalid method")
	}

	return &MetricBuilder{
		url:     url,
		method:  Method,
		metrics: make([]MetricMetricBuilder, 0),
	}, nil
}
Example #17
0
/*
registerIPs does the actual work of determining the IPs on the host. Parameters are the hostId for this host
and the function used to send the found IPs
*/
func registerIPs(hostId string, sendFn SendHostIPs) error {

	interfaces, err := net.Interfaces()
	if err != nil {
		glog.Error("Problem reading interfaces: ", err)
		return err
	}
	hostIPResources := make([]dao.HostIPResource, 0, len(interfaces))
	for _, iface := range interfaces {
		addrs, err := iface.Addrs()
		if err != nil {
			glog.Errorf("Problem reading address for interface %s: %s", iface.Name, err)
			return err
		}
		for _, addr := range addrs {
			//send address to Master
			hostIp := dao.HostIPResource{}
			hostIp.IPAddress = addr.String()
			hostIp.InterfaceName = iface.Name
			hostIPResources = append(hostIPResources, hostIp)
		}
	}
	var unused int
	glog.V(4).Infof("Agent registering IPs %v", hostIPResources)
	hostIps := dao.HostIPs{}
	hostIps.HostId = hostId
	hostIps.IPs = hostIPResources
	if err := sendFn(hostIps, &unused); err != nil {
		glog.Errorf("Error registering IPs %v", err)
	}
	return nil
}
Example #18
0
func RestGetService(w *rest.ResponseWriter, r *rest.Request, client *serviced.ControlClient) {
	var allServices []*dao.Service

	if err := client.GetServices(&empty, &allServices); err != nil {
		glog.Errorf("Could not get services: %v", err)
		RestServerError(w)
		return
	}

	sid, err := url.QueryUnescape(r.PathParam("serviceId"))
	if err != nil {
		RestBadRequest(w)
		return
	}

	for _, service := range allServices {
		if service.Id == sid {
			w.WriteJson(&service)
			return
		}
	}

	glog.Errorf("No such service [%v]", sid)
	RestServerError(w)
}
Example #19
0
// restRemovePoolVirtualIP deletes virtualip
func restRemovePoolVirtualIP(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	ip, err := url.QueryUnescape(r.PathParam("ip"))
	if err != nil {
		glog.Errorf("Could not get virtual ip (%v): %v", ip, err)
		restBadRequest(w, err)
		return
	}

	poolId, err := url.QueryUnescape(r.PathParam("poolId"))
	if err != nil {
		glog.Errorf("Could not get virtual ip poolId (%v): %v", poolId, err)
		restBadRequest(w, err)
		return
	}

	glog.V(0).Infof("Remove virtual ip=%v (in pool %v)", ip, poolId)

	client, err := ctx.getMasterClient()
	if err != nil {
		restServerError(w, err)
		return
	}

	request := pool.VirtualIP{PoolID: poolId, IP: ip}
	if err := client.RemoveVirtualIP(request); err != nil {
		glog.Errorf("Failed to remove virtual IP(%+v): %v", request, err)
		restServerError(w, err)
		return
	}
	restSuccess(w)
}
Example #20
0
// Retrieve service container port info.
func (ss *ServiceState) GetHostEndpointInfo(applicationRegex *regexp.Regexp) (hostPort, containerPort uint16, protocol string, match bool) {
	for _, ep := range ss.Endpoints {

		if ep.Purpose == "export" {
			if applicationRegex.MatchString(ep.Application) {
				if ep.PortTemplate != "" {
					port, err := ss.evalPortTemplate(ep.PortTemplate)
					if err != nil {
						glog.Errorf("%+v", err)
						break
					}
					ep.PortNumber = uint16(port)
				}
				portS := fmt.Sprintf("%d/%s", ep.PortNumber, strings.ToLower(ep.Protocol))

				external := ss.PortMapping[portS]
				if len(external) == 0 {
					glog.Warningf("Found match for %s:%s, but no portmapping is available", applicationRegex, portS)
					break
				}

				extPort, err := strconv.ParseUint(external[0].HostPort, 10, 16)
				if err != nil {
					glog.Errorf("Portmap parsing failed for %s:%s %v", applicationRegex, portS, err)
					break
				}
				return uint16(extPort), ep.PortNumber, ep.Protocol, true
			}
		}
	}

	return 0, 0, "", false
}
Example #21
0
func (ed *elasticDriver) getHealth() (map[string]interface{}, error) {
	health := make(map[string]interface{})
	healthURL := fmt.Sprintf("%v/_cluster/health", ed.elasticURL())
	resp, err := http.Get(healthURL)
	if err != nil {
		return health, err
	}
	if resp.StatusCode != 200 {
		return health, fmt.Errorf("http status: %v", resp.StatusCode)
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		glog.Errorf("error reading elastic health: %v", err)
		return health, err
	}
	if err := json.Unmarshal(body, &health); err != nil {
		glog.Errorf("error unmarshalling elastic health: %v; err: %v", string(body), err)
		return health, err
	}
	glog.V(4).Infof("Elastic Health: %v; err: %v", string(body), err)
	return health, nil

}
Example #22
0
func (viface *vif) redirectCommand(from, to, protocol string) error {
	glog.Infof("Trying to set up redirect %s:%s->:%s %s", viface.hostname, from, to, protocol)
	for _, chain := range []string{"OUTPUT", "PREROUTING"} {
		command := []string{
			"iptables",
			"-t", "nat",
			"-A", chain,
			"-d", viface.ip,
			"-p", protocol,
			"--dport", from,
			"-j", "REDIRECT",
			"--to-ports", to,
		}
		c := exec.Command(command[0], command[1:]...)

		c.Stdout = os.Stdout
		c.Stderr = os.Stdout
		if err := c.Run(); err != nil {
			glog.Errorf("Unable to set up redirect %s:%s->:%s %s command:%+v", viface.hostname, from, to, protocol, command)
			return err
		}
	}

	glog.Infof("AddToEtcHosts(%s, %s)", viface.hostname, viface.ip)
	err := node.AddToEtcHosts(viface.hostname, viface.ip)
	if err != nil {
		glog.Errorf("Unable to add %s %s to /etc/hosts", viface.ip, viface.hostname)
		return err
	}
	return nil
}
Example #23
0
func restGetService(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
	sid, err := url.QueryUnescape(r.PathParam("serviceId"))
	if err != nil {
		restBadRequest(w, err)
		return
	}

	if strings.Contains(sid, "isvc-") {
		w.WriteJson(isvcs.ISVCSMap[sid])
		return
	}
	svc := service.Service{}
	if err := client.GetService(sid, &svc); err != nil {
		glog.Errorf("Could not get service %v: %v", sid, err)
		restServerError(w, err)
		return
	}

	if svc.ID == sid {
		fillBuiltinMetrics(&svc)
		w.WriteJson(&svc)
		return
	}

	glog.Errorf("No such service [%v]", sid)
	restServerError(w, err)
}
Example #24
0
func (dfs *DistributedFilesystem) importImages(dirpath string, images []imagemeta, tenants map[string]struct{}) error {
	for _, metadata := range images {
		filename := filepath.Join(dirpath, metadata.Filename)

		// Make sure all images that refer to a local registry are named with the local registry
		tags := make([]string, len(metadata.Tags))
		for i, tag := range metadata.Tags {
			imageID, err := commons.ParseImageID(tag)
			if err != nil {
				glog.Errorf("Could not parse %s: %s", tag, err)
				return err
			}

			if _, ok := tenants[imageID.User]; ok {
				imageID.Host, imageID.Port = dfs.dockerHost, dfs.dockerPort
			}
			tags[i] = imageID.String()
		}

		if err := loadImage(filename, metadata.UUID, tags); err != nil {
			glog.Errorf("Error loading %s (%s): %s", filename, metadata.UUID, err)
			return err
		}
	}
	return nil
}
Example #25
0
func (svc *IService) start() (<-chan int, error) {
	ctr, err := svc.attach()
	if err != nil {
		svc.setStoppedHealthStatus(fmt.Errorf("could not start service: %s", err))
		return nil, err
	}

	// destroy the container when it dies
	notify := make(chan int, 1)
	ctr.OnEvent(docker.Die, func(cid string) { svc.remove(notify) })

	// start the container
	if err := ctr.Start(10 * time.Second); err != nil && err != docker.ErrAlreadyStarted {
		svc.setStoppedHealthStatus(fmt.Errorf("could not start service: %s", err))
		return nil, err
	}

	// perform an initial healthcheck to verify that the service started successfully
	select {
	case err := <-svc.startupHealthcheck():
		if err != nil {
			glog.Errorf("Healthcheck for %s failed: %s", svc.Name, err)
			svc.stop()
			return nil, err
		}
		svc.startTime = time.Now()
	case rc := <-notify:
		glog.Errorf("isvc %s exited on startup, rc=%d", svc.Name, rc)
		svc.setStoppedHealthStatus(ExitError(rc))
		return nil, ExitError(rc)
	}

	return notify, nil
}
Example #26
0
func (this *ControlPlaneDao) GetRunningService(request dao.ServiceStateRequest, running *dao.RunningService) error {
	glog.V(3).Infof("ControlPlaneDao.GetRunningService: request=%v", request)
	*running = dao.RunningService{}

	serviceID := request.ServiceID
	poolID, err := this.facade.GetPoolForService(datastore.Get(), serviceID)
	if err != nil {
		glog.Errorf("Unable to get service %v: %v", serviceID, err)
		return err
	}

	poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID))
	if err != nil {
		glog.Errorf("Error in getting a connection based on pool %v: %v", poolID, err)
		return err
	}

	if thisRunning, err := zkservice.LoadRunningService(poolBasedConn, request.ServiceID, request.ServiceStateID); err != nil {
		glog.Errorf("zkservice.LoadRunningService failed (conn: %+v serviceID: %v): %v", poolBasedConn, request.ServiceID, err)
		return err
	} else {
		if thisRunning != nil {
			*running = *thisRunning
		}
	}

	return nil
}
Example #27
0
// Snapshots returns the current snapshots on the volume (sorted by date)
func (c *BtrfsConn) Snapshots() ([]string, error) {
	c.Lock()
	defer c.Unlock()

	glog.V(2).Infof("listing snapshots of volume:%v and c.name:%s ", c.root, c.name)
	output, err := runcmd(c.sudoer, "subvolume", "list", "-s", c.root)
	if err != nil {
		glog.Errorf("Could not list subvolumes of %s: %s", c.root, err)
		return nil, err
	}

	var files []os.FileInfo
	for _, line := range strings.Split(string(output), "\n") {
		glog.V(2).Infof("line: %s", line)
		if parts := strings.Split(line, "path"); len(parts) == 2 {
			label := strings.TrimSpace(parts[1])
			label = strings.TrimPrefix(label, "volumes/")
			glog.V(2).Infof("looking for tenant:%s in label:'%s'", c.name, label)
			if strings.HasPrefix(label, c.name+"_") {
				file, err := os.Stat(filepath.Join(c.root, label))
				if err != nil {
					glog.Errorf("Could not stat snapshot %s: %s", label, err)
					return nil, err
				}
				files = append(files, file)
				glog.V(2).Infof("found snapshot:%s", label)
			}
		}
	}

	return volume.FileInfoSlice(files).Labels(), nil
}
Example #28
0
func (this *ControlPlaneDao) GetRunningServicesForHost(hostID string, services *[]dao.RunningService) error {
	// we initialize the data container to something here in case it has not been initialized yet
	*services = make([]dao.RunningService, 0)
	myHost, err := this.facade.GetHost(datastore.Get(), hostID)
	if err != nil {
		glog.Errorf("Unable to get host %v: %v", hostID, err)
		return err
	} else if myHost == nil {
		return nil
	}

	poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(myHost.PoolID))
	if err != nil {
		glog.Errorf("Error in getting a connection based on pool %v: %v", myHost.PoolID, err)
		return err
	}

	*services, err = zkservice.LoadRunningServicesByHost(poolBasedConn, hostID)
	if err != nil {
		glog.Errorf("zkservice.LoadRunningServicesByHost (conn: %+v host: %v) failed: %v", poolBasedConn, hostID, err)
		return err
	}

	return nil
}
Example #29
0
// Spawn starts the remote Synchronizer based on nodeID
func (l *Synchronizer) Spawn(shutdown <-chan interface{}, nodeID string) {
	// Start dependent listeners
	var (
		wg        sync.WaitGroup
		_shutdown = make(chan interface{})
		stopping  = make(chan interface{})
	)

	defer func() {
		close(_shutdown)
		wg.Wait()
	}()

	var id string
	var wait <-chan time.Time
	for {
		node := l.Allocate()
		event, err := l.conn.GetW(l.GetPath(nodeID), node)
		if err == client.ErrNoNode && id != "" {
			if err := l.Delete(id); err != nil {
				glog.Errorf("Could not delete node at %s: %s", l.GetPath(nodeID), err)
				wait = time.After(DefaultRetryTime)
			} else {
				return
			}
		} else if err != nil {
			glog.Errorf("Could not get node at %s: %s", l.GetPath(nodeID), err)
			return
		}

		if key, err := l.AddUpdate(id, node); err == ErrInvalidType {
			glog.Errorf("Invalid type detected")
			return
		} else if err != nil {
			glog.Errorf("Could not update node at %s: %s", l.GetPath(nodeID), err)
			wait = time.After(DefaultRetryTime)
		} else if id == "" {
			wg.Add(1)
			go func() {
				defer wg.Done()
				if len(l.lfuncs) == 0 {
					return
				}
				defer close(stopping)
				l.startListeners(_shutdown, nodeID)
			}()
			id = key
		}

		select {
		case <-event:
		case <-wait:
		case <-stopping:
			return
		case <-shutdown:
			return
		}
	}
}
Example #30
0
func (f *Facade) getAutoAssignment(ctx datastore.Context, poolID string, ports ...uint16) (ipinfo, error) {
	pool, err := f.GetResourcePool(ctx, poolID)
	if err != nil {
		glog.Errorf("Error while looking up pool %s: %s", poolID, err)
		return ipinfo{}, err
	}

	ignoreips := make(map[string]struct{})
	for _, port := range ports {
		// Get all of the address assignments for port
		assignments, err := f.GetServiceAddressAssignmentsByPort(ctx, port)
		if err != nil {
			glog.Errorf("Error while looking up address assignments for port %d: %s", port, err)
			return ipinfo{}, err
		}

		// Find out all of the host ips that cannot be used
		for _, assignment := range assignments {
			ignoreips[assignment.IPAddr] = struct{}{}
		}
	}

	// Filter virtual ips
	var ips []ipinfo
	for _, vip := range pool.VirtualIPs {
		if _, ok := ignoreips[vip.IP]; !ok {
			ips = append(ips, ipinfo{vip.IP, commons.VIRTUAL, ""})
		}
	}

	hosts, err := f.FindHostsInPool(ctx, poolID)
	if err != nil {
		glog.Errorf("Error while looking up hosts in pool %s: %s", poolID, err)
		return ipinfo{}, err
	}
	var resources []host.HostIPResource
	for _, host := range hosts {
		if host.IPs != nil {
			resources = append(resources, host.IPs...)
		}
	}
	// Filter static ips
	for _, hostIP := range resources {
		if _, ok := ignoreips[hostIP.IPAddress]; !ok {
			ips = append(ips, ipinfo{hostIP.IPAddress, commons.STATIC, hostIP.HostID})
		}
	}

	// Pick an ip
	total := len(ips)
	if total == 0 {
		err := fmt.Errorf("No IPs are available to be assigned")
		glog.Errorf("Error acquiring IP assignment: %s", err)
		return ipinfo{}, err
	}

	rand.Seed(time.Now().UTC().UnixNano())
	return ips[rand.Intn(total)], nil
}