// CreateProxyCmds creates the commands needed for a proxy unit. func (e *dockerEngine) CreateProxyCmds(t *jobs.Task, link jobs.Link, linkIndex int, env map[string]string, scalingGroup uint) (engine.Cmds, error) { containerName := fmt.Sprintf("%s-pr%d", t.ContainerName(scalingGroup), linkIndex) containerImage := images.Wormhole execStart, err := e.createProxyDockerCmdLine(t, containerName, containerImage, link, env, scalingGroup) if err != nil { return engine.Cmds{}, maskAny(err) } var cmds engine.Cmds cmds.Start = append(cmds.Start, e.pullCmd(containerImage), e.stopCmd(containerName), e.removeCmd(containerName), e.cleanupCmd(), ) if e.options.EnvFile != "" { cmds.Start = append(cmds.Start, *cmdline.New(nil, e.touchPath, e.options.EnvFile)) } cmds.Start = append(cmds.Start, execStart) cmds.Stop = append(cmds.Stop, e.stopCmd(containerName), e.removeCmd(containerName), ) return cmds, nil }
// unitDescription creates the description of a unit func unitDescription(t *jobs.Task, prefix string, scalingGroup uint) string { descriptionPostfix := fmt.Sprintf("[slice %d]", scalingGroup) if t.GroupGlobal() { descriptionPostfix = "[global]" } return fmt.Sprintf("%s unit for %s %s", prefix, t.FullName(), descriptionPostfix) }
// createMainUnit func createMainUnit(t *jobs.Task, sidekickUnitNames []string, engine engine.Engine, ctx generatorContext) (*sdunits.Unit, error) { unit, err := createDefaultUnit(t, unitName(t, unitKindMain, ctx.ScalingGroup), unitDescription(t, "Main", ctx.ScalingGroup), "service", unitKindMain, ctx) if err != nil { return nil, maskAny(err) } cmds, err := engine.CreateMainCmds(t, unit.ExecOptions.Environment, ctx.ScalingGroup) if err != nil { return nil, maskAny(err) } setupUnitFromCmds(unit, cmds) switch t.Type { case "oneshot": unit.ExecOptions.IsOneshot = true unit.ExecOptions.Restart = "on-failure" case "proxy": unit.ExecOptions.Restart = "always" default: unit.ExecOptions.Restart = "always" } // Additional service dependencies unit.ExecOptions.Require(sidekickUnitNames...) unit.ExecOptions.After(sidekickUnitNames...) for _, name := range t.VolumesFrom { other, err := t.Task(name) if err != nil { return nil, maskAny(err) } otherName := unitName(other, unitKindMain, ctx.ScalingGroup) + ".service" unit.ExecOptions.Require(otherName) unit.ExecOptions.After(otherName) } // Add metrics registration commands if err := addMetricsRegistration(t, unit, ctx); err != nil { return nil, maskAny(err) } // Add frontend registration commands if err := addFrontEndRegistration(t, unit, ctx); err != nil { return nil, maskAny(err) } return unit, nil }
// CreateMainCmds creates the commands needed for a main unit. func (e *dockerEngine) CreateMainCmds(t *jobs.Task, env map[string]string, scalingGroup uint) (engine.Cmds, error) { containerName := t.ContainerName(scalingGroup) containerImage := t.Image.String() if t.Type == "proxy" { containerImage = images.Alpine } execStart, err := e.createMainDockerCmdLine(t, containerImage, env, scalingGroup) if err != nil { return engine.Cmds{}, maskAny(err) } var cmds engine.Cmds cmds.Start = append(cmds.Start, e.pullCmd(containerImage), ) if e.options.EnvFile != "" { cmds.Start = append(cmds.Start, *cmdline.New(nil, e.touchPath, e.options.EnvFile)) } // Add secret extraction commands secretsCmds, err := e.createSecretsExecStartPre(t, images.VaultMonkey, env, scalingGroup) if err != nil { return engine.Cmds{}, maskAny(err) } cmds.Start = append(cmds.Start, secretsCmds...) cmds.Start = append(cmds.Start, e.stopCmd(containerName), e.removeCmd(containerName), e.cleanupCmd(), ) for _, v := range t.Volumes { if v.IsLocal() { cmds.Start = append(cmds.Start, e.createTestLocalVolumeCmd(v.HostPath)) } } cmds.Start = append(cmds.Start, execStart) cmds.Stop = append(cmds.Stop, e.stopCmd(containerName), e.removeCmd(containerName), ) return cmds, nil }
// addDockerNetworkArgs adds docker network arguments for the given task. func (e *dockerEngine) addDockerNetworkArgs(c *cmdline.Cmdline, env map[string]string, t *jobs.Task) error { switch t.Network { case "", jobs.NetworkTypeDefault: return nil case jobs.NetworkTypeHost: c.Add(env, "--net=host") return nil case jobs.NetworkTypeWeave: if !t.Type.IsProxy() { c.Add(env, fmt.Sprintf("--hostname=%s", t.WeaveDomainName())) } else { c.Add(env, fmt.Sprintf("--hostname=%s", t.PrivateDomainName())) } return nil default: return maskAny(fmt.Errorf("Unknown network type '%s", t.Network)) } }
// createVolumeUnit func createVolumeUnit(t *jobs.Task, vol jobs.Volume, volIndex int, engine engine.Engine, ctx generatorContext) (*sdunits.Unit, error) { namePostfix := createVolumeUnitNamePostfix(volIndex) volPrefix := path.Join(fmt.Sprintf("%s/%v", t.FullName(), ctx.ScalingGroup), vol.Path) volHostPath := fmt.Sprintf("/media/%s", volPrefix) unit, err := createDefaultUnit(t, unitName(t, namePostfix, ctx.ScalingGroup), unitDescription(t, fmt.Sprintf("Volume %d", volIndex), ctx.ScalingGroup), "service", namePostfix, ctx) if err != nil { return nil, maskAny(err) } cmds, err := engine.CreateVolumeCmds(t, vol, volIndex, volPrefix, volHostPath, unit.ExecOptions.Environment, ctx.ScalingGroup) if err != nil { return nil, maskAny(err) } setupUnitFromCmds(unit, cmds) return unit, nil }
// unitNameExt returns the name of the systemd unit for this task. func unitNameExt(t *jobs.Task, kind string, scalingGroup string) string { base := strings.Replace(t.FullName(), "/", "-", -1) + kind if t.GroupGlobal() && t.GroupCount() == 1 { return base } return fmt.Sprintf("%s@%v", base, scalingGroup) }
// addFrontEndRegistration adds registration code for frontends to the given units func addFrontEndRegistration(t *jobs.Task, main *sdunits.Unit, ctx generatorContext) error { if len(t.PublicFrontEnds) == 0 && len(t.PrivateFrontEnds) == 0 { return nil } serviceName := t.ServiceName() targetServiceName := serviceName if t.Type == "proxy" { targetServiceName = t.Target.EtcdServiceName() } httpKey := fmt.Sprintf("/pulcy/frontend/%s-%d", serviceName, ctx.ScalingGroup) httpRecord := api.FrontendRecord{ Service: targetServiceName, HttpCheckPath: t.HttpCheckPath, HttpCheckMethod: t.HttpCheckMethod, Sticky: t.Sticky, Backup: t.Backup, Mode: "", // Defaults to http } tcpKey := fmt.Sprintf("/pulcy/frontend/%s-%d-tcp", serviceName, ctx.ScalingGroup) tcpRecord := api.FrontendRecord{ Service: targetServiceName, HttpCheckPath: t.HttpCheckPath, HttpCheckMethod: t.HttpCheckMethod, Sticky: t.Sticky, Backup: t.Backup, Mode: "tcp", } instanceHttpKey := fmt.Sprintf("/pulcy/frontend/%s-%d-inst", serviceName, ctx.ScalingGroup) instanceHttpRecord := api.FrontendRecord{ Service: fmt.Sprintf("%s-%d", targetServiceName, ctx.ScalingGroup), HttpCheckPath: t.HttpCheckPath, Sticky: t.Sticky, Backup: t.Backup, } instanceTcpKey := fmt.Sprintf("/pulcy/frontend/%s-%d-inst-tcp", serviceName, ctx.ScalingGroup) instanceTcpRecord := api.FrontendRecord{ Service: fmt.Sprintf("%s-%d", targetServiceName, ctx.ScalingGroup), HttpCheckPath: t.HttpCheckPath, Sticky: t.Sticky, Backup: t.Backup, Mode: "tcp", } var rwRules []api.RewriteRule if t.Type == "proxy" && t.Rewrite != nil { rwRules = append(rwRules, api.RewriteRule{ PathPrefix: t.Rewrite.PathPrefix, RemovePathPrefix: t.Rewrite.RemovePathPrefix, Domain: t.Rewrite.Domain, }) } for _, fr := range t.PublicFrontEnds { record := &httpRecord if fr.Mode == "tcp" { record = &tcpRecord } selRecord := api.FrontendSelectorRecord{ Weight: fr.Weight, Domain: fr.Domain, PathPrefix: fr.PathPrefix, SslCert: fr.SslCert, ServicePort: fr.Port, FrontendPort: fr.HostPort, RewriteRules: rwRules, } if err := addUsers(t, &selRecord, fr.Users); err != nil { return maskAny(err) } record.Selectors = append(record.Selectors, selRecord) } for _, fr := range t.PrivateFrontEnds { record := &httpRecord instanceRecord := &instanceHttpRecord if fr.Mode == "tcp" { record = &tcpRecord instanceRecord = &instanceTcpRecord } selRecord := api.FrontendSelectorRecord{ Domain: t.PrivateDomainName(), ServicePort: fr.Port, FrontendPort: fr.HostPort, Private: true, RewriteRules: rwRules, } if err := addUsers(t, &selRecord, fr.Users); err != nil { return maskAny(err) } record.Selectors = append(record.Selectors, selRecord) if fr.RegisterInstance { instanceSelRecord := selRecord instanceSelRecord.Domain = t.InstanceSpecificPrivateDomainName(ctx.ScalingGroup) instanceRecord.Selectors = append(instanceRecord.Selectors, instanceSelRecord) } } if len(instanceHttpRecord.Selectors) > 0 { if err := addFrontEndRegistrationRecord(t, main, instanceHttpKey, instanceHttpRecord, "FrontEndRegistration-i"); err != nil { return maskAny(err) } } if len(instanceTcpRecord.Selectors) > 0 { if err := addFrontEndRegistrationRecord(t, main, instanceTcpKey, instanceTcpRecord, "FrontEndRegistration-i-tcp"); err != nil { return maskAny(err) } } if len(httpRecord.Selectors) > 0 { if err := addFrontEndRegistrationRecord(t, main, httpKey, httpRecord, "FrontEndRegistration"); err != nil { return maskAny(err) } } if len(tcpRecord.Selectors) > 0 { if err := addFrontEndRegistrationRecord(t, main, tcpKey, tcpRecord, "FrontEndRegistration-tcp"); err != nil { return maskAny(err) } } return nil }
// createMainDockerCmdLine creates the `ExecStart` line for // the main unit. func (e *dockerEngine) createMainDockerCmdLine(t *jobs.Task, image string, env map[string]string, scalingGroup uint) (cmdline.Cmdline, error) { serviceName := t.ServiceName() cmd, err := e.createDockerCmd(env, t.Network) if err != nil { return cmd, maskAny(err) } cmd.Add(nil, "run", "--rm", fmt.Sprintf("--name %s", t.ContainerName(scalingGroup))) if err := e.addDockerNetworkArgs(&cmd, env, t); err != nil { return cmd, maskAny(err) } if len(t.Ports) > 0 { for _, p := range t.Ports { cmd.Add(env, fmt.Sprintf("-p %s", p)) } } else { cmd.Add(env, "-P") } for i, v := range t.Volumes { if v.IsLocal() { cmd.Add(env, fmt.Sprintf("-v %s", v)) } else if !v.IsLocal() { cmd.Add(env, fmt.Sprintf("--volumes-from %s", createVolumeUnitContainerName(t, i, scalingGroup))) } } for _, secret := range t.Secrets { if ok, path := secret.TargetFile(); ok { hostPath, err := secretFilePath(t, scalingGroup, secret) if err != nil { return cmdline.Cmdline{}, maskAny(err) } cmd.Add(env, fmt.Sprintf("-v %s:%s:ro", hostPath, path)) } } for _, name := range t.VolumesFrom { other, err := t.Task(name) if err != nil { return cmdline.Cmdline{}, maskAny(err) } for i, v := range other.Volumes { if !v.IsLocal() { cmd.Add(env, fmt.Sprintf("--volumes-from %s", createVolumeUnitContainerName(other, i, scalingGroup))) } } cmd.Add(env, fmt.Sprintf("--volumes-from %s", other.ContainerName(scalingGroup))) } envKeys := []string{} for k := range t.Environment { envKeys = append(envKeys, k) } sort.Strings(envKeys) if e.options.EnvFile != "" { cmd.Add(env, fmt.Sprintf("--env-file=%s", e.options.EnvFile)) } for _, k := range envKeys { cmd.Add(env, "-e "+strconv.Quote(fmt.Sprintf("%s=%s", k, t.Environment[k]))) } if t.Secrets.AnyTargetEnviroment() { cmd.Add(env, "--env-file="+secretEnvironmentPath(t, scalingGroup)) } cmd.Add(env, fmt.Sprintf("-e SERVICE_NAME=%s", serviceName)) // Support registrator for _, cap := range t.Capabilities { cmd.Add(env, "--cap-add "+cap) } tcpLinkIndex := 0 for _, l := range t.Links { targetName := l.Target.PrivateDomainName() if l.Type.IsHTTP() { cmd.Add(env, "--add-host") cmd.Add(env, fmt.Sprintf("%s:${COREOS_PRIVATE_IPV4}", targetName)) } else { linkContainerName := fmt.Sprintf("%s-pr%d", t.ContainerName(scalingGroup), tcpLinkIndex) cmd.Add(env, fmt.Sprintf("--link %s:%s", linkContainerName, targetName)) tcpLinkIndex++ } } for _, arg := range t.LogDriver.CreateDockerLogArgs(e.options) { cmd.Add(env, arg) } for _, arg := range t.DockerArgs { cmd.Add(env, arg) } if t.User != "" { cmd.Add(env, fmt.Sprintf("--user %s", t.User)) } cmd.Add(nil, image) if t.Type == "proxy" { cmd.Add(nil, "sleep 36500d") } for _, arg := range t.Args { cmd.Add(env, arg) } return cmd, nil }
// createVolumeUnitContainerName creates the name of the docker container that serves a volume with given index func createVolumeUnitContainerName(t *jobs.Task, volIndex int, scalingGroup uint) string { return fmt.Sprintf("%s-vl%d", t.ContainerName(scalingGroup), volIndex) }
// createSecretsUnit creates a unit used to extract secrets from vault func (e *dockerEngine) createSecretsExecStartPre(t *jobs.Task, containerImage string, env map[string]string, scalingGroup uint) ([]cmdline.Cmdline, error) { if len(t.Secrets) == 0 { // No secrets to extract return nil, nil } // Create all secret extraction commands jobID := t.JobID() if jobID == "" { return nil, maskAny(fmt.Errorf("job ID missing for job %s with secrets", t.JobName())) } // Prepare volume paths secretsRoot := secretsRootPath(t, scalingGroup) secretsRootVol := fmt.Sprintf("%s:%s", secretsRoot, secretsRoot) vaultCrtVol := "/etc/pulcy/vault.crt:/etc/pulcy/vault.crt:ro" clusterIdVol := "/etc/pulcy/cluster-id:/etc/pulcy/cluster-id:ro" machineIdVol := "/etc/machine-id:/etc/machine-id:ro" var cmds []cmdline.Cmdline cmds = append(cmds, *cmdline.New(nil, "/usr/bin/mkdir", "-p", secretsRoot), e.pullCmd(containerImage), ) envPaths := []string{} for _, secret := range t.Secrets { if ok, _ := secret.TargetFile(); ok { targetPath, err := secretFilePath(t, scalingGroup, secret) if err != nil { return nil, maskAny(err) } var cmd cmdline.Cmdline cmd.Add(nil, e.dockerPath, "run", "--rm") //cmd.Add(env, fmt.Sprintf("--name %s-sc", t.containerName(ctx.ScalingGroup))) cmd.Add(env, "--net=host") cmd.Add(env, "-v "+secretsRootVol) cmd.Add(env, "-v "+vaultCrtVol) cmd.Add(env, "-v "+clusterIdVol) cmd.Add(env, "-v "+machineIdVol) cmd.Add(env, "--env-file /etc/pulcy/vault.env") /*if ctx.DockerOptions.EnvFile != "" { cmd.Add(env,fmt.Sprintf("--env-file=%s", ctx.DockerOptions.EnvFile)) }*/ for _, arg := range t.LogDriver.CreateDockerLogArgs(e.options) { cmd.Add(env, arg) } cmd.Add(env, containerImage) cmd.Add(nil, "extract", "file") cmd.Add(env, "--target "+targetPath) cmd.Add(env, "--job-id "+jobID) cmd.Add(env, secret.VaultPath()) cmds = append(cmds, cmd) } else if ok, environmentKey := secret.TargetEnviroment(); ok { envPaths = append(envPaths, fmt.Sprintf("%s=%s", environmentKey, secret.VaultPath())) } } if len(envPaths) > 0 { targetPath := secretEnvironmentPath(t, scalingGroup) var cmd cmdline.Cmdline cmd.Add(nil, e.dockerPath, "run", "--rm") //cmd.Add(env, fmt.Sprintf("--name %s-sc", t.containerName(ctx.ScalingGroup))) cmd.Add(env, "--net=host") cmd.Add(env, "-v "+secretsRootVol) cmd.Add(env, "-v "+vaultCrtVol) cmd.Add(env, "-v "+clusterIdVol) cmd.Add(env, "-v "+machineIdVol) cmd.Add(env, "--env-file /etc/pulcy/vault.env") /*if ctx.DockerOptions.EnvFile != "" { cmd.Add(env, fmt.Sprintf("--env-file=%s", ctx.DockerOptions.EnvFile)) }*/ for _, arg := range t.LogDriver.CreateDockerLogArgs(e.options) { cmd.Add(env, arg) } cmd.Add(env, containerImage) cmd.Add(nil, "extract", "env") cmd.Add(env, "--target "+targetPath) cmd.Add(env, "--job-id "+jobID) for _, envPath := range envPaths { cmd.Add(env, envPath) } cmds = append(cmds, cmd) } return cmds, nil }
// secretsRootPath returns the path of the root directory that will contain secret files for the given task. func secretsRootPath(t *jobs.Task, scalingGroup uint) string { return filepath.Join(secretsPath, t.ContainerName(scalingGroup)) }
// CreateFleetAfter creates a list of unit names to add to the `After` setting of each unit of the given task. func addFleetOptions(t *jobs.Task, fleetOptions cluster.FleetOptions, unit *sdunits.Unit) { jobName := t.JobName() unit.ExecOptions.After(excludeUnitsOfJob(jobName, fleetOptions.After)...) unit.ExecOptions.Want(excludeUnitsOfJob(jobName, fleetOptions.Wants)...) unit.ExecOptions.Require(excludeUnitsOfJob(jobName, fleetOptions.Requires)...) }