func ports(c *project.ServiceConfig) (map[dockerclient.Port]struct{}, map[dockerclient.Port][]dockerclient.PortBinding, error) {
ports, binding, err := nat.ParsePortSpecs(c.Ports)
if err != nil {
return nil, nil, err
}
exPorts, _, err := nat.ParsePortSpecs(c.Expose)
if err != nil {
return nil, nil, err
}
for k, v := range exPorts {
ports[k] = v
}
exposedPorts := map[dockerclient.Port]struct{}{}
for k, v := range ports {
exposedPorts[dockerclient.Port(k)] = v
}
portBindings := map[dockerclient.Port][]dockerclient.PortBinding{}
for k, bv := range binding {
dcbs := make([]dockerclient.PortBinding, len(bv))
for k, v := range bv {
dcbs[k] = dockerclient.PortBinding{HostIP: v.HostIP, HostPort: v.HostPort}
}
portBindings[dockerclient.Port(k)] = dcbs
}
return exposedPorts, portBindings, nil
}
func parsePort(port string) docker.Port {
if port == "" {
return docker.Port(port)
}
if strings.Contains(port, "/") {
return docker.Port(port)
} else {
return docker.Port(port + "/tcp")
}
}
func CreateContainer(containerName, image, domain string, portsToMap []Port, environmentVars map[string]string) (*docker.Container, error) {
cfg := &docker.Config{}
cfg.Image = image
cfg.Hostname = containerName
cfg.Domainname = domain
var e struct{}
exposedPorts := make(map[docker.Port]struct{})
for _, p := range portsToMap {
prt := strconv.FormatUint(p.Private, 10) + "/" + p.Type
exposedPorts[docker.Port(prt)] = e
fmt.Printf("Exposing %s\n", prt)
}
cfg.ExposedPorts = exposedPorts
envStrs := make([]string, 0, 10)
for k, v := range environmentVars {
envStrs = append(envStrs, k+"="+v)
}
cfg.Env = envStrs
hostCfg := &docker.HostConfig{}
hostCfg.PublishAllPorts = false
hostCfg.Privileged = false
hostPorts := make(map[docker.Port][]docker.PortBinding)
for _, p := range portsToMap {
prt := strconv.FormatUint(p.Private, 10) + "/" + p.Type
bindings := make([]docker.PortBinding, 0, 4)
bindings = append(bindings, docker.PortBinding{HostIP: "", HostPort: strconv.FormatUint(p.Public, 10)})
fmt.Printf("Binding %s to %s\n", prt, bindings[0])
hostPorts[docker.Port(prt)] = bindings
}
hostCfg.PortBindings = hostPorts
opts := docker.CreateContainerOptions{}
opts.Config = cfg
opts.Name = containerName
opts.HostConfig = hostCfg
json, _ := data.PrettyPrint(opts)
fmt.Printf("create options: %s\n", string(json))
container, err := client.CreateContainer(opts)
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating container: %s\n", err.Error())
} else {
fmt.Printf("Container created: %s\n", container.ID)
}
return container, err
}
func asPortBindings(ports []string) (map[docker.Port][]docker.PortBinding, map[docker.Port]struct{}) {
binds := make(map[docker.Port][]docker.PortBinding)
exposed := make(map[docker.Port]struct{})
for _, port := range ports {
parts := strings.SplitN(port, ":", 2)
proto, cport := nat.SplitProtoPort(parts[1])
cport = cport + "/" + proto
binds[docker.Port(cport)] = []docker.PortBinding{{HostPort: parts[0]}}
exposed[docker.Port(cport)] = struct{}{}
}
return binds, exposed
}
func portSetToDockerPorts(ports *schema.Set) (map[dc.Port]struct{}, map[dc.Port][]dc.PortBinding) {
retExposedPorts := map[dc.Port]struct{}{}
retPortBindings := map[dc.Port][]dc.PortBinding{}
for _, portInt := range ports.List() {
port := portInt.(map[string]interface{})
internal := port["internal"].(int)
protocol := port["protocol"].(string)
exposedPort := dc.Port(strconv.Itoa(internal) + "/" + protocol)
retExposedPorts[exposedPort] = struct{}{}
external, extOk := port["external"].(int)
ip, ipOk := port["ip"].(string)
if extOk {
portBinding := dc.PortBinding{
HostPort: strconv.Itoa(external),
}
if ipOk {
portBinding.HostIP = ip
}
retPortBindings[exposedPort] = append(retPortBindings[exposedPort], portBinding)
}
}
return retExposedPorts, retPortBindings
}
func CreateCassandraAndStart(name, port string) (*docker.Container, error) {
endpoint := "unix:///var/run/docker.sock"
client, err := docker.NewClient(endpoint)
if err != nil {
return nil, err
}
hostConf := &docker.HostConfig{
PortBindings: map[docker.Port][]docker.PortBinding{
docker.Port("9042/tcp"): []docker.PortBinding{{
HostIP: "0.0.0.0",
HostPort: port,
}},
},
}
/*hostConf := &docker.HostConfig{
PublishAllPorts: true,
}*/
opts := docker.CreateContainerOptions{
Name: name,
Config: &docker.Config{
Image: "spotify/cassandra:latest",
},
HostConfig: hostConf,
}
container, err := client.CreateContainer(opts)
if err != nil {
return nil, err
}
return container, client.StartContainer(container.ID, hostConf)
}
func TestDockerDriver_PortsMapping(t *testing.T) {
task, res, dyn := dockerTask()
task.Config["port_map"] = []map[string]string{
map[string]string{
"main": "8080",
"REDIS": "6379",
},
}
client, handle, cleanup := dockerSetup(t, task)
defer cleanup()
waitForExist(t, client, handle.(*DockerHandle))
container, err := client.InspectContainer(handle.(*DockerHandle).ContainerID())
if err != nil {
t.Fatalf("err: %v", err)
}
// Verify that the correct ports are EXPOSED
expectedExposedPorts := map[docker.Port]struct{}{
docker.Port("8080/tcp"): struct{}{},
docker.Port("8080/udp"): struct{}{},
docker.Port("6379/tcp"): struct{}{},
docker.Port("6379/udp"): struct{}{},
}
if !reflect.DeepEqual(container.Config.ExposedPorts, expectedExposedPorts) {
t.Errorf("Exposed ports don't match.\nExpected:\n%s\nGot:\n%s\n", expectedExposedPorts, container.Config.ExposedPorts)
}
// Verify that the correct ports are FORWARDED
expectedPortBindings := map[docker.Port][]docker.PortBinding{
docker.Port("8080/tcp"): []docker.PortBinding{docker.PortBinding{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", res)}},
docker.Port("8080/udp"): []docker.PortBinding{docker.PortBinding{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", res)}},
docker.Port("6379/tcp"): []docker.PortBinding{docker.PortBinding{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", dyn)}},
docker.Port("6379/udp"): []docker.PortBinding{docker.PortBinding{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", dyn)}},
}
if !reflect.DeepEqual(container.HostConfig.PortBindings, expectedPortBindings) {
t.Errorf("Forwarded ports don't match.\nExpected:\n%s\nGot:\n%s\n", expectedPortBindings, container.HostConfig.PortBindings)
}
expectedEnvironment := map[string]string{
"NOMAD_ADDR_main": "127.0.0.1:8080",
"NOMAD_ADDR_REDIS": "127.0.0.1:6379",
"NOMAD_HOST_PORT_main": strconv.Itoa(docker_reserved),
}
for key, val := range expectedEnvironment {
search := fmt.Sprintf("%s=%s", key, val)
if !inSlice(search, container.Config.Env) {
t.Errorf("Expected to find %s in container environment: %+v", search, container.Config.Env)
}
}
}
func TestDockerPortsNoMap(t *testing.T) {
t.Parallel()
task, res, dyn := dockerTask()
client, handle, cleanup := dockerSetup(t, task)
defer cleanup()
container, err := client.InspectContainer(handle.(*DockerHandle).ContainerID())
if err != nil {
t.Fatalf("err: %v", err)
}
// Verify that the correct ports are EXPOSED
expectedExposedPorts := map[docker.Port]struct{}{
docker.Port(fmt.Sprintf("%d/tcp", res)): struct{}{},
docker.Port(fmt.Sprintf("%d/udp", res)): struct{}{},
docker.Port(fmt.Sprintf("%d/tcp", dyn)): struct{}{},
docker.Port(fmt.Sprintf("%d/udp", dyn)): struct{}{},
// This one comes from the redis container
docker.Port("6379/tcp"): struct{}{},
}
if !reflect.DeepEqual(container.Config.ExposedPorts, expectedExposedPorts) {
t.Errorf("Exposed ports don't match.\nExpected:\n%s\nGot:\n%s\n", expectedExposedPorts, container.Config.ExposedPorts)
}
// Verify that the correct ports are FORWARDED
expectedPortBindings := map[docker.Port][]docker.PortBinding{
docker.Port(fmt.Sprintf("%d/tcp", res)): []docker.PortBinding{docker.PortBinding{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", res)}},
docker.Port(fmt.Sprintf("%d/udp", res)): []docker.PortBinding{docker.PortBinding{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", res)}},
docker.Port(fmt.Sprintf("%d/tcp", dyn)): []docker.PortBinding{docker.PortBinding{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", dyn)}},
docker.Port(fmt.Sprintf("%d/udp", dyn)): []docker.PortBinding{docker.PortBinding{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", dyn)}},
}
if !reflect.DeepEqual(container.HostConfig.PortBindings, expectedPortBindings) {
t.Errorf("Forwarded ports don't match.\nExpected:\n%s\nGot:\n%s\n", expectedPortBindings, container.HostConfig.PortBindings)
}
expectedEnvironment := map[string]string{
"NOMAD_ADDR_main": fmt.Sprintf("127.0.0.1:%d", res),
"NOMAD_ADDR_REDIS": fmt.Sprintf("127.0.0.1:%d", dyn),
}
for key, val := range expectedEnvironment {
search := fmt.Sprintf("%s=%s", key, val)
if !inSlice(search, container.Config.Env) {
t.Errorf("Expected to find %s in container environment: %+v", search, container.Config.Env)
}
}
}
// StartContainer starts a test container and returns its ID.
func StartContainer(daemon *dockertest.Docker, bindings []*beacon.Binding, labels map[string]string) (string, error) {
client, err := daemon.Client()
if err != nil {
return "", errors.Wrap(err, "unable to create docker client")
}
exposedPorts := map[dockerclient.Port]struct{}{}
for _, binding := range bindings {
port := dockerclient.Port(fmt.Sprintf("%d/%s", binding.ContainerPort, binding.Protocol))
exposedPorts[port] = struct{}{}
}
container, err := client.CreateContainer(dockerclient.CreateContainerOptions{
Config: &dockerclient.Config{
Image: fmt.Sprintf("%s:%s", DOCKER_IMAGE_REPO, DOCKER_IMAGE_TAG),
Cmd: []string{"/bin/sh", "-c", "trap exit SIGTERM SIGINT; while true; do sleep 0.1; done"},
ExposedPorts: exposedPorts,
Labels: labels,
},
})
if err != nil {
return "", errors.Wrap(err, "unable to create docker container")
}
portBindings := map[dockerclient.Port][]dockerclient.PortBinding{}
for _, binding := range bindings {
port := dockerclient.Port(fmt.Sprintf("%d/%s", binding.ContainerPort, binding.Protocol))
portBindings[port] = []dockerclient.PortBinding{{
HostIP: binding.HostIP,
HostPort: fmt.Sprintf("%d", binding.HostPort),
}}
}
err = client.StartContainer(container.ID, &dockerclient.HostConfig{
PortBindings: portBindings,
})
if err != nil {
StopContainer(daemon, container.ID)
return "", errors.Wrap(err, "unable to start docker container")
}
return container.ID, nil
}
// createContainer initializes a struct needed to call docker.client.CreateContainer()
func createContainer(ctx *ExecContext, task *structs.Task, logger *log.Logger) docker.CreateContainerOptions {
if task.Resources == nil {
panic("task.Resources is nil and we can't constrain resource usage. We shouldn't have been able to schedule this in the first place.")
}
hostConfig := createHostConfig(task)
logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Config["image"])
logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Config["image"])
// Setup port mapping (equivalent to -p on docker CLI). Ports must already be
// exposed in the container.
if len(task.Resources.Networks) == 0 {
logger.Print("[WARN] driver.docker: No networks are available for port mapping")
} else {
network := task.Resources.Networks[0]
dockerPorts := map[docker.Port][]docker.PortBinding{}
for _, port := range network.ListStaticPorts() {
dockerPorts[docker.Port(strconv.Itoa(port)+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}}
dockerPorts[docker.Port(strconv.Itoa(port)+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}}
logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static) %s\n", network.IP, port, port)
}
for label, port := range network.MapDynamicPorts() {
// If the label is numeric we expect that there is a service
// listening on that port inside the container. In this case we'll
// setup a mapping from our random host port to the label port.
//
// Otherwise we'll setup a direct 1:1 mapping from the host port to
// the container, and assume that the process inside will read the
// environment variable and bind to the correct port.
if _, err := strconv.Atoi(label); err == nil {
dockerPorts[docker.Port(label+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}}
dockerPorts[docker.Port(label+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}}
logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %s (mapped)", network.IP, port, label)
} else {
dockerPorts[docker.Port(strconv.Itoa(port)+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}}
dockerPorts[docker.Port(strconv.Itoa(port)+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}}
logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d for label %s\n", network.IP, port, port, label)
}
}
hostConfig.PortBindings = dockerPorts
}
config := &docker.Config{
Env: TaskEnvironmentVariables(ctx, task).List(),
Image: task.Config["image"],
}
// If the user specified a custom command to run, we'll inject it here.
if command, ok := task.Config["command"]; ok {
config.Cmd = strings.Split(command, " ")
}
return docker.CreateContainerOptions{
Config: config,
HostConfig: hostConfig,
}
}
// EXPOSE 6667/tcp 7000/tcp
//
// Expose ports for links and port mappings. This all ends up in
// b.RunConfig.ExposedPorts for runconfig.
//
func expose(b *Builder, args []string, attributes map[string]bool, original string) error {
if len(args) == 0 {
return errAtLeastOneArgument("EXPOSE")
}
if b.RunConfig.ExposedPorts == nil {
b.RunConfig.ExposedPorts = make(map[docker.Port]struct{})
}
existing := map[string]struct{}{}
for k := range b.RunConfig.ExposedPorts {
existing[k.Port()] = struct{}{}
}
for _, port := range args {
dp := docker.Port(port)
if _, exists := existing[dp.Port()]; !exists {
b.RunConfig.ExposedPorts[docker.Port(fmt.Sprintf("%s/%s", dp.Port(), dp.Proto()))] = struct{}{}
}
}
return nil
}
/*
Extract a "mapped port" address. This mode assumes the balancer is
connecting to containers via a port "mapped" (NATed) by
Docker. Therefore it looks for the port mentioned in the list of
published ports, and finds the host port it has been mapped to. The IP
address is that given as the host's IP address.
*/
func (l *Listener) mappedPortAddress(container *docker.Container, port int) (string, int) {
p := docker.Port(fmt.Sprintf("%d/tcp", port))
if bindings, found := container.NetworkSettings.Ports[p]; found {
for _, binding := range bindings {
if binding.HostIP == l.hostIP || binding.HostIP == "" || binding.HostIP == "0.0.0.0" {
mappedToPort, err := strconv.Atoi(binding.HostPort)
if err != nil {
return "", 0
}
return l.hostIP, mappedToPort
}
}
}
return "", 0
}
// Port returns the host port the specified port is mapped on.
func (c *Container) Port(port string) (string, error) {
info, err := c.findInfo()
if err != nil {
return "", err
}
if bindings, ok := info.NetworkSettings.Ports[dockerclient.Port(port)]; ok {
result := []string{}
for _, binding := range bindings {
result = append(result, binding.HostIP+":"+binding.HostPort)
}
return strings.Join(result, "\n"), nil
}
return "", nil
}
func (h *Runner) PortForward(local, remote int) *Runner {
if h.hostConfig.PortBindings == nil {
h.hostConfig.PortBindings = map[docker.Port][]docker.PortBinding{}
}
containerPort := docker.Port(fmt.Sprintf("%d/tcp", remote))
binding := docker.PortBinding{
HostPort: fmt.Sprintf("%d", local),
}
h.hostConfig.PortBindings[containerPort] = []docker.PortBinding{binding}
if h.config.ExposedPorts == nil {
h.config.ExposedPorts = map[docker.Port]struct{}{}
}
h.config.ExposedPorts[containerPort] = struct{}{}
return h
}
func (m *mockInspector) startContainers(cs ...container) {
for _, c := range cs {
env := []string{}
for k, v := range c.Env {
env = append(env, strings.Join([]string{k, v}, "="))
}
ports := map[docker.Port][]docker.PortBinding{}
for k, v := range c.Ports {
ports[docker.Port(k)] = []docker.PortBinding{
docker.PortBinding{
HostIP: "0.0.0.0",
HostPort: v,
},
}
}
netmode := c.NetworkMode
if netmode == "" {
netmode = "default"
}
c1 := &docker.Container{
ID: c.ID,
HostConfig: &docker.HostConfig{
NetworkMode: netmode,
},
Config: &docker.Config{
Image: c.Image,
Env: env,
Labels: c.Labels,
},
NetworkSettings: &docker.NetworkSettings{
IPAddress: c.IPAddress,
Ports: ports,
},
}
m.containers[c.ID] = c1
if m.events != nil {
m.events <- &docker.APIEvents{
Status: "start",
ID: c.ID,
}
}
}
}
// LookupPort lookup the given port for a container and return the first port value.
// It matches based on the IP of the caller.
func (d *DockerDiscovery) LookupPort(ip, port string) (int, error) {
// default to TCP if not specified.
if strings.Index(port, "/") == -1 {
port += "/tcp"
}
log := logrus.WithField("port", port).WithField("ip", ip)
// small return helper.
returnPort := func(cont *docker.Container) (int, error) {
ports, ok := cont.NetworkSettings.Ports[docker.Port(port)]
if !ok {
log.Warning("The port is not exposed")
return -1, nil
}
port, err := strconv.Atoi(strings.Split(ports[0].HostPort, "/")[0])
if err != nil {
log.WithError(err).Error("Invalid port format")
return -1, err
}
return port, nil
}
// Lookup all containers.
containers, err := d.client.ListContainers(docker.ListContainersOptions{All: false})
if err != nil {
return -1, err
}
for _, cont := range containers {
// Fetch more details about that container.
cont, err := d.client.InspectContainer(cont.ID)
if err != nil {
log.WithError(err).Error("error inspecting container, skipping")
continue
}
// If the hostname and IP match, we return the port.
if cont.NetworkSettings.IPAddress == ip {
return returnPort(cont)
}
}
// If we reach this point, we didn't find the container.
return -1, fmt.Errorf("unable to lookup the port %s for container %s", port, ip)
}
// FuzzerFor can randomly populate api objects that are destined for version.
func FuzzerFor(t *testing.T, version unversioned.GroupVersion, src rand.Source) *fuzz.Fuzzer {
f := fuzz.New().NilChance(.5).NumElements(1, 1)
if src != nil {
f.RandSource(src)
}
f.Funcs(
func(j *int, c fuzz.Continue) {
*j = int(c.Int31())
},
func(j **int, c fuzz.Continue) {
if c.RandBool() {
i := int(c.Int31())
*j = &i
} else {
*j = nil
}
},
func(q *resource.Quantity, c fuzz.Continue) {
*q = *resource.NewQuantity(c.Int63n(1000), resource.DecimalExponent)
},
func(j *runtime.TypeMeta, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = ""
j.Kind = ""
},
func(j *unversioned.TypeMeta, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = ""
j.Kind = ""
},
func(j *api.ObjectMeta, c fuzz.Continue) {
j.Name = c.RandString()
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.SelfLink = c.RandString()
j.UID = types.UID(c.RandString())
j.GenerateName = c.RandString()
var sec, nsec int64
c.Fuzz(&sec)
c.Fuzz(&nsec)
j.CreationTimestamp = unversioned.Unix(sec, nsec).Rfc3339Copy()
},
func(j *api.ObjectReference, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = c.RandString()
j.Kind = c.RandString()
j.Namespace = c.RandString()
j.Name = c.RandString()
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.FieldPath = c.RandString()
},
func(j *unversioned.ListMeta, c fuzz.Continue) {
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.SelfLink = c.RandString()
},
func(j *api.ListOptions, c fuzz.Continue) {
label, _ := labels.Parse("a=b")
j.LabelSelector = label
field, _ := fields.ParseSelector("a=b")
j.FieldSelector = field
},
func(j *api.PodExecOptions, c fuzz.Continue) {
j.Stdout = true
j.Stderr = true
},
func(j *api.PodAttachOptions, c fuzz.Continue) {
j.Stdout = true
j.Stderr = true
},
func(s *api.PodSpec, c fuzz.Continue) {
c.FuzzNoCustom(s)
// has a default value
ttl := int64(30)
if c.RandBool() {
ttl = int64(c.Uint32())
}
s.TerminationGracePeriodSeconds = &ttl
c.Fuzz(s.SecurityContext)
if s.SecurityContext == nil {
s.SecurityContext = new(api.PodSecurityContext)
}
},
func(j *api.PodPhase, c fuzz.Continue) {
statuses := []api.PodPhase{api.PodPending, api.PodRunning, api.PodFailed, api.PodUnknown}
*j = statuses[c.Rand.Intn(len(statuses))]
},
func(j *api.Binding, c fuzz.Continue) {
c.Fuzz(&j.ObjectMeta)
j.Target.Name = c.RandString()
},
func(j *api.ReplicationControllerSpec, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
//j.TemplateRef = nil // this is required for round trip
},
func(j *extensions.DeploymentStrategy, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
// Ensure that strategyType is one of valid values.
strategyTypes := []extensions.DeploymentStrategyType{extensions.RecreateDeploymentStrategyType, extensions.RollingUpdateDeploymentStrategyType}
j.Type = strategyTypes[c.Rand.Intn(len(strategyTypes))]
if j.Type != extensions.RollingUpdateDeploymentStrategyType {
j.RollingUpdate = nil
} else {
rollingUpdate := extensions.RollingUpdateDeployment{}
if c.RandBool() {
rollingUpdate.MaxUnavailable = intstr.FromInt(int(c.RandUint64()))
rollingUpdate.MaxSurge = intstr.FromInt(int(c.RandUint64()))
} else {
rollingUpdate.MaxSurge = intstr.FromString(fmt.Sprintf("%d%%", c.RandUint64()))
}
j.RollingUpdate = &rollingUpdate
}
},
func(j *extensions.JobSpec, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
completions := int(c.Rand.Int31())
parallelism := int(c.Rand.Int31())
j.Completions = &completions
j.Parallelism = ¶llelism
if c.Rand.Int31()%2 == 0 {
j.ManualSelector = newBool(true)
} else {
j.ManualSelector = nil
}
},
func(j *api.List, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
// TODO: uncomment when round trip starts from a versioned object
if false { //j.Items == nil {
j.Items = []runtime.Object{}
}
},
func(j *runtime.Object, c fuzz.Continue) {
// TODO: uncomment when round trip starts from a versioned object
if true { //c.RandBool() {
*j = &runtime.Unknown{
// We do not set TypeMeta here because it is not carried through a round trip
Raw: []byte(`{"apiVersion":"unknown.group/unknown","kind":"Something","someKey":"someValue"}`),
ContentType: runtime.ContentTypeJSON,
}
} else {
types := []runtime.Object{&api.Pod{}, &api.ReplicationController{}}
t := types[c.Rand.Intn(len(types))]
c.Fuzz(t)
*j = t
}
},
func(pb map[docker.Port][]docker.PortBinding, c fuzz.Continue) {
// This is necessary because keys with nil values get omitted.
// TODO: Is this a bug?
pb[docker.Port(c.RandString())] = []docker.PortBinding{
{c.RandString(), c.RandString()},
{c.RandString(), c.RandString()},
}
},
func(pm map[string]docker.PortMapping, c fuzz.Continue) {
// This is necessary because keys with nil values get omitted.
// TODO: Is this a bug?
pm[c.RandString()] = docker.PortMapping{
c.RandString(): c.RandString(),
}
},
func(q *api.ResourceRequirements, c fuzz.Continue) {
randomQuantity := func() resource.Quantity {
var q resource.Quantity
c.Fuzz(&q)
return q
}
q.Limits = make(api.ResourceList)
q.Requests = make(api.ResourceList)
cpuLimit := randomQuantity()
q.Limits[api.ResourceCPU] = *cpuLimit.Copy()
q.Requests[api.ResourceCPU] = *cpuLimit.Copy()
memoryLimit := randomQuantity()
q.Limits[api.ResourceMemory] = *memoryLimit.Copy()
q.Requests[api.ResourceMemory] = *memoryLimit.Copy()
storageLimit := randomQuantity()
q.Limits[api.ResourceStorage] = *storageLimit.Copy()
q.Requests[api.ResourceStorage] = *storageLimit.Copy()
},
func(q *api.LimitRangeItem, c fuzz.Continue) {
var cpuLimit resource.Quantity
c.Fuzz(&cpuLimit)
q.Type = api.LimitTypeContainer
q.Default = make(api.ResourceList)
q.Default[api.ResourceCPU] = *(cpuLimit.Copy())
q.DefaultRequest = make(api.ResourceList)
q.DefaultRequest[api.ResourceCPU] = *(cpuLimit.Copy())
q.Max = make(api.ResourceList)
q.Max[api.ResourceCPU] = *(cpuLimit.Copy())
q.Min = make(api.ResourceList)
q.Min[api.ResourceCPU] = *(cpuLimit.Copy())
q.MaxLimitRequestRatio = make(api.ResourceList)
q.MaxLimitRequestRatio[api.ResourceCPU] = resource.MustParse("10")
},
func(p *api.PullPolicy, c fuzz.Continue) {
policies := []api.PullPolicy{api.PullAlways, api.PullNever, api.PullIfNotPresent}
*p = policies[c.Rand.Intn(len(policies))]
},
func(rp *api.RestartPolicy, c fuzz.Continue) {
policies := []api.RestartPolicy{api.RestartPolicyAlways, api.RestartPolicyNever, api.RestartPolicyOnFailure}
*rp = policies[c.Rand.Intn(len(policies))]
},
// Only api.DownwardAPIVolumeFile needs to have a specific func since FieldRef has to be
// defaulted to a version otherwise roundtrip will fail
// For the remaining volume plugins the default fuzzer is enough.
func(m *api.DownwardAPIVolumeFile, c fuzz.Continue) {
m.Path = c.RandString()
versions := []string{"v1"}
m.FieldRef.APIVersion = versions[c.Rand.Intn(len(versions))]
m.FieldRef.FieldPath = c.RandString()
},
func(vs *api.VolumeSource, c fuzz.Continue) {
// Exactly one of the fields must be set.
v := reflect.ValueOf(vs).Elem()
i := int(c.RandUint64() % uint64(v.NumField()))
t := v.Field(i).Addr()
for v.Field(i).IsNil() {
c.Fuzz(t.Interface())
}
},
func(i *api.ISCSIVolumeSource, c fuzz.Continue) {
i.ISCSIInterface = c.RandString()
if i.ISCSIInterface == "" {
i.ISCSIInterface = "default"
}
},
func(d *api.DNSPolicy, c fuzz.Continue) {
policies := []api.DNSPolicy{api.DNSClusterFirst, api.DNSDefault}
*d = policies[c.Rand.Intn(len(policies))]
},
func(p *api.Protocol, c fuzz.Continue) {
protocols := []api.Protocol{api.ProtocolTCP, api.ProtocolUDP}
*p = protocols[c.Rand.Intn(len(protocols))]
},
func(p *api.ServiceAffinity, c fuzz.Continue) {
types := []api.ServiceAffinity{api.ServiceAffinityClientIP, api.ServiceAffinityNone}
*p = types[c.Rand.Intn(len(types))]
},
func(p *api.ServiceType, c fuzz.Continue) {
types := []api.ServiceType{api.ServiceTypeClusterIP, api.ServiceTypeNodePort, api.ServiceTypeLoadBalancer}
*p = types[c.Rand.Intn(len(types))]
},
func(ct *api.Container, c fuzz.Continue) {
c.FuzzNoCustom(ct) // fuzz self without calling this function again
ct.TerminationMessagePath = "/" + ct.TerminationMessagePath // Must be non-empty
},
func(p *api.Probe, c fuzz.Continue) {
c.FuzzNoCustom(p)
// These fields have default values.
intFieldsWithDefaults := [...]string{"TimeoutSeconds", "PeriodSeconds", "SuccessThreshold", "FailureThreshold"}
v := reflect.ValueOf(p).Elem()
for _, field := range intFieldsWithDefaults {
f := v.FieldByName(field)
if f.Int() == 0 {
f.SetInt(1)
}
}
},
func(ev *api.EnvVar, c fuzz.Continue) {
ev.Name = c.RandString()
if c.RandBool() {
ev.Value = c.RandString()
} else {
ev.ValueFrom = &api.EnvVarSource{}
ev.ValueFrom.FieldRef = &api.ObjectFieldSelector{}
var versions []unversioned.GroupVersion
for _, testGroup := range testapi.Groups {
versions = append(versions, *testGroup.GroupVersion())
}
ev.ValueFrom.FieldRef.APIVersion = versions[c.Rand.Intn(len(versions))].String()
ev.ValueFrom.FieldRef.FieldPath = c.RandString()
}
},
func(sc *api.SecurityContext, c fuzz.Continue) {
v1beta3GroupVersion := unversioned.GroupVersion{Group: "", Version: "v1beta3"}
c.FuzzNoCustom(sc) // fuzz self without calling this function again
if c.RandBool() || version == v1beta3GroupVersion {
priv := c.RandBool()
sc.Privileged = &priv
}
if c.RandBool() || version == v1beta3GroupVersion {
sc.Capabilities = &api.Capabilities{
Add: make([]api.Capability, 0),
Drop: make([]api.Capability, 0),
}
c.Fuzz(&sc.Capabilities.Add)
c.Fuzz(&sc.Capabilities.Drop)
}
},
func(s *api.Secret, c fuzz.Continue) {
c.FuzzNoCustom(s) // fuzz self without calling this function again
s.Type = api.SecretTypeOpaque
},
func(pv *api.PersistentVolume, c fuzz.Continue) {
c.FuzzNoCustom(pv) // fuzz self without calling this function again
types := []api.PersistentVolumePhase{api.VolumeAvailable, api.VolumePending, api.VolumeBound, api.VolumeReleased, api.VolumeFailed}
pv.Status.Phase = types[c.Rand.Intn(len(types))]
pv.Status.Message = c.RandString()
reclamationPolicies := []api.PersistentVolumeReclaimPolicy{api.PersistentVolumeReclaimRecycle, api.PersistentVolumeReclaimRetain}
pv.Spec.PersistentVolumeReclaimPolicy = reclamationPolicies[c.Rand.Intn(len(reclamationPolicies))]
},
func(pvc *api.PersistentVolumeClaim, c fuzz.Continue) {
c.FuzzNoCustom(pvc) // fuzz self without calling this function again
types := []api.PersistentVolumeClaimPhase{api.ClaimBound, api.ClaimPending}
pvc.Status.Phase = types[c.Rand.Intn(len(types))]
},
func(s *api.NamespaceSpec, c fuzz.Continue) {
s.Finalizers = []api.FinalizerName{api.FinalizerKubernetes}
},
func(s *api.NamespaceStatus, c fuzz.Continue) {
s.Phase = api.NamespaceActive
},
func(http *api.HTTPGetAction, c fuzz.Continue) {
c.FuzzNoCustom(http) // fuzz self without calling this function again
http.Path = "/" + http.Path // can't be blank
http.Scheme = "x" + http.Scheme // can't be blank
},
func(ss *api.ServiceSpec, c fuzz.Continue) {
c.FuzzNoCustom(ss) // fuzz self without calling this function again
if len(ss.Ports) == 0 {
// There must be at least 1 port.
ss.Ports = append(ss.Ports, api.ServicePort{})
c.Fuzz(&ss.Ports[0])
}
for i := range ss.Ports {
switch ss.Ports[i].TargetPort.Type {
case intstr.Int:
ss.Ports[i].TargetPort.IntVal = 1 + ss.Ports[i].TargetPort.IntVal%65535 // non-zero
case intstr.String:
ss.Ports[i].TargetPort.StrVal = "x" + ss.Ports[i].TargetPort.StrVal // non-empty
}
}
},
func(n *api.Node, c fuzz.Continue) {
c.FuzzNoCustom(n)
n.Spec.ExternalID = "external"
},
func(s *api.NodeStatus, c fuzz.Continue) {
c.FuzzNoCustom(s)
s.Allocatable = s.Capacity
},
func(s *extensions.HorizontalPodAutoscalerSpec, c fuzz.Continue) {
c.FuzzNoCustom(s) // fuzz self without calling this function again
minReplicas := int(c.Rand.Int31())
s.MinReplicas = &minReplicas
s.CPUUtilization = &extensions.CPUTargetUtilization{TargetPercentage: int(int32(c.RandUint64()))}
},
func(s *extensions.SubresourceReference, c fuzz.Continue) {
c.FuzzNoCustom(s) // fuzz self without calling this function again
s.Subresource = "scale"
},
func(psp *extensions.PodSecurityPolicySpec, c fuzz.Continue) {
c.FuzzNoCustom(psp) // fuzz self without calling this function again
runAsUserRules := []extensions.RunAsUserStrategy{extensions.RunAsUserStrategyMustRunAsNonRoot, extensions.RunAsUserStrategyMustRunAs, extensions.RunAsUserStrategyRunAsAny}
psp.RunAsUser.Rule = runAsUserRules[c.Rand.Intn(len(runAsUserRules))]
seLinuxRules := []extensions.SELinuxStrategy{extensions.SELinuxStrategyRunAsAny, extensions.SELinuxStrategyMustRunAs}
psp.SELinux.Rule = seLinuxRules[c.Rand.Intn(len(seLinuxRules))]
},
func(s *extensions.Scale, c fuzz.Continue) {
c.FuzzNoCustom(s) // fuzz self without calling this function again
// TODO: Implement a fuzzer to generate valid keys, values and operators for
// selector requirements.
if s.Status.Selector != nil {
s.Status.Selector = &unversioned.LabelSelector{
MatchLabels: map[string]string{
"testlabelkey": "testlabelval",
},
MatchExpressions: []unversioned.LabelSelectorRequirement{
{
Key: "testkey",
Operator: unversioned.LabelSelectorOpIn,
Values: []string{"val1", "val2", "val3"},
},
},
}
}
},
func(scc *api.SecurityContextConstraints, c fuzz.Continue) {
c.FuzzNoCustom(scc) // fuzz self without calling this function again
userTypes := []api.RunAsUserStrategyType{api.RunAsUserStrategyMustRunAsNonRoot, api.RunAsUserStrategyMustRunAs, api.RunAsUserStrategyRunAsAny, api.RunAsUserStrategyMustRunAsRange}
scc.RunAsUser.Type = userTypes[c.Rand.Intn(len(userTypes))]
seLinuxTypes := []api.SELinuxContextStrategyType{api.SELinuxStrategyRunAsAny, api.SELinuxStrategyMustRunAs}
scc.SELinuxContext.Type = seLinuxTypes[c.Rand.Intn(len(seLinuxTypes))]
supGroupTypes := []api.SupplementalGroupsStrategyType{api.SupplementalGroupsStrategyMustRunAs, api.SupplementalGroupsStrategyRunAsAny}
scc.SupplementalGroups.Type = supGroupTypes[c.Rand.Intn(len(supGroupTypes))]
fsGroupTypes := []api.FSGroupStrategyType{api.FSGroupStrategyMustRunAs, api.FSGroupStrategyRunAsAny}
scc.FSGroup.Type = fsGroupTypes[c.Rand.Intn(len(fsGroupTypes))]
// when fuzzing the volume types ensure it is set to avoid the defaulter's expansion.
// Do not use FSTypeAll or host dir setting to steer clear of defaulting mechanics
// which are covered in specific unit tests.
volumeTypes := []api.FSType{api.FSTypeAWSElasticBlockStore,
api.FSTypeAzureFile,
api.FSTypeCephFS,
api.FSTypeCinder,
api.FSTypeDownwardAPI,
api.FSTypeEmptyDir,
api.FSTypeFC,
api.FSTypeFlexVolume,
api.FSTypeFlocker,
api.FSTypeGCEPersistentDisk,
api.FSTypeGitRepo,
api.FSTypeGlusterfs,
api.FSTypeISCSI,
api.FSTypeNFS,
api.FSTypePersistentVolumeClaim,
api.FSTypeRBD,
api.FSTypeSecret}
scc.Volumes = []api.FSType{volumeTypes[c.Rand.Intn(len(volumeTypes))]}
},
)
return f
}
// createContainer initializes a struct needed to call docker.client.CreateContainer()
func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, driverConfig *DockerDriverConfig) (docker.CreateContainerOptions, error) {
var c docker.CreateContainerOptions
if task.Resources == nil {
// Guard against missing resources. We should never have been able to
// schedule a job without specifying this.
d.logger.Println("[ERR] driver.docker: task.Resources is empty")
return c, fmt.Errorf("task.Resources is empty")
}
binds, err := d.containerBinds(ctx.AllocDir, task)
if err != nil {
return c, err
}
// Create environment variables.
env := TaskEnvironmentVariables(ctx, task)
env.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName))
env.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal))
config := &docker.Config{
Image: driverConfig.ImageName,
Hostname: driverConfig.Hostname,
}
hostConfig := &docker.HostConfig{
// Convert MB to bytes. This is an absolute value.
//
// This value represents the total amount of memory a process can use.
// Swap is added to total memory and is managed by the OS, not docker.
// Since this may cause other processes to swap and cause system
// instability, we will simply not use swap.
//
// See: https://www.kernel.org/doc/Documentation/cgroups/memory.txt
Memory: int64(task.Resources.MemoryMB) * 1024 * 1024,
MemorySwap: -1,
// Convert Mhz to shares. This is a relative value.
//
// There are two types of CPU limiters available: Shares and Quotas. A
// Share allows a particular process to have a proportion of CPU time
// relative to other processes; 1024 by default. A CPU Quota is enforced
// over a Period of time and is a HARD limit on the amount of CPU time a
// process can use. Processes with quotas cannot burst, while processes
// with shares can, so we'll use shares.
//
// The simplest scale is 1 share to 1 MHz so 1024 = 1GHz. This means any
// given process will have at least that amount of resources, but likely
// more since it is (probably) rare that the machine will run at 100%
// CPU. This scale will cease to work if a node is overprovisioned.
//
// See:
// - https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
// - https://www.kernel.org/doc/Documentation/scheduler/sched-design-CFS.txt
CPUShares: int64(task.Resources.CPU),
// Binds are used to mount a host volume into the container. We mount a
// local directory for storage and a shared alloc directory that can be
// used to share data between different tasks in the same task group.
Binds: binds,
}
d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Config["image"])
d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Config["image"])
d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Config["image"])
// set privileged mode
hostPrivileged := d.config.ReadBoolDefault("docker.privileged.enabled", false)
if driverConfig.Privileged && !hostPrivileged {
return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent`)
}
hostConfig.Privileged = hostPrivileged
// set DNS servers
for _, ip := range driverConfig.DNSServers {
if net.ParseIP(ip) != nil {
hostConfig.DNS = append(hostConfig.DNS, ip)
} else {
d.logger.Printf("[ERR] driver.docker: invalid ip address for container dns server: %s", ip)
}
}
// set DNS search domains
for _, domain := range driverConfig.DNSSearchDomains {
hostConfig.DNSSearch = append(hostConfig.DNSSearch, domain)
}
hostConfig.NetworkMode = driverConfig.NetworkMode
if hostConfig.NetworkMode == "" {
// docker default
d.logger.Println("[DEBUG] driver.docker: networking mode not specified; defaulting to bridge")
hostConfig.NetworkMode = "bridge"
}
// Setup port mapping and exposed ports
if len(task.Resources.Networks) == 0 {
d.logger.Println("[DEBUG] driver.docker: No network interfaces are available")
if len(driverConfig.PortMap) > 0 {
return c, fmt.Errorf("Trying to map ports but no network interface is available")
}
} else {
// TODO add support for more than one network
network := task.Resources.Networks[0]
publishedPorts := map[docker.Port][]docker.PortBinding{}
exposedPorts := map[docker.Port]struct{}{}
for _, port := range network.ReservedPorts {
// By default we will map the allocated port 1:1 to the container
containerPortInt := port.Value
// If the user has mapped a port using port_map we'll change it here
if mapped, ok := driverConfig.PortMap[port.Label]; ok {
containerPortInt = mapped
}
hostPortStr := strconv.Itoa(port.Value)
containerPort := docker.Port(strconv.Itoa(containerPortInt))
publishedPorts[containerPort+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}}
publishedPorts[containerPort+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}}
d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)", network.IP, port.Value, port.Value)
exposedPorts[containerPort+"/tcp"] = struct{}{}
exposedPorts[containerPort+"/udp"] = struct{}{}
d.logger.Printf("[DEBUG] driver.docker: exposed port %d", port.Value)
}
for _, port := range network.DynamicPorts {
// By default we will map the allocated port 1:1 to the container
containerPortInt := port.Value
// If the user has mapped a port using port_map we'll change it here
if mapped, ok := driverConfig.PortMap[port.Label]; ok {
containerPortInt = mapped
}
hostPortStr := strconv.Itoa(port.Value)
containerPort := docker.Port(strconv.Itoa(containerPortInt))
publishedPorts[containerPort+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}}
publishedPorts[containerPort+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}}
d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (mapped)", network.IP, port.Value, containerPortInt)
exposedPorts[containerPort+"/tcp"] = struct{}{}
exposedPorts[containerPort+"/udp"] = struct{}{}
d.logger.Printf("[DEBUG] driver.docker: exposed port %s", containerPort)
}
// This was set above in a call to TaskEnvironmentVariables but if we
// have mapped any ports we will need to override them.
//
// TODO refactor the implementation in TaskEnvironmentVariables to match
// the 0.2 ports world view. Docker seems to be the only place where
// this is actually needed, but this is kinda hacky.
if len(driverConfig.PortMap) > 0 {
env.SetPorts(network.MapLabelToValues(driverConfig.PortMap))
}
hostConfig.PortBindings = publishedPorts
config.ExposedPorts = exposedPorts
}
parsedArgs := args.ParseAndReplace(driverConfig.Args, env.Map())
// If the user specified a custom command to run as their entrypoint, we'll
// inject it here.
if driverConfig.Command != "" {
cmd := []string{driverConfig.Command}
if len(driverConfig.Args) != 0 {
cmd = append(cmd, parsedArgs...)
}
d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s", strings.Join(cmd, " "))
config.Cmd = cmd
} else if len(driverConfig.Args) != 0 {
d.logger.Println("[DEBUG] driver.docker: ignoring command arguments because command is not specified")
}
if len(driverConfig.Labels) > 0 {
config.Labels = driverConfig.Labels
d.logger.Printf("[DEBUG] driver.docker: applied labels on the container: %+v", config.Labels)
}
config.Env = env.List()
containerName := fmt.Sprintf("%s-%s", task.Name, ctx.AllocID)
d.logger.Printf("[DEBUG] driver.docker: setting container name to: %s", containerName)
return docker.CreateContainerOptions{
Name: containerName,
Config: config,
HostConfig: hostConfig,
}, nil
}
defer m.RUnlock()
for _, c := range m.events {
c <- event
}
}
var (
container1 = &client.Container{
ID: "ping",
Name: "pong",
Image: "baz",
State: client.State{Pid: 1, Running: true},
NetworkSettings: &client.NetworkSettings{
IPAddress: "1.2.3.4",
Ports: map[client.Port][]client.PortBinding{
client.Port("80/tcp"): {
{
HostIP: "1.2.3.4",
HostPort: "80",
},
},
client.Port("81/tcp"): {},
},
},
Config: &client.Config{
Labels: map[string]string{
"foo1": "bar1",
"foo2": "bar2",
},
},
}
// createAndStartRouterContainer is responsible for deploying the router image in docker. It assumes that all router images
// will use a command line flag that can take --master which points to the master url
func createAndStartRouterContainer(dockerCli *dockerClient.Client, masterIp string, routerStatsPort int) (containerId string, err error) {
ports := []string{"80", "443"}
if routerStatsPort > 0 {
ports = append(ports, fmt.Sprintf("%d", routerStatsPort))
}
portBindings := make(map[dockerClient.Port][]dockerClient.PortBinding)
exposedPorts := map[dockerClient.Port]struct{}{}
for _, p := range ports {
dockerPort := dockerClient.Port(p + "/tcp")
portBindings[dockerPort] = []dockerClient.PortBinding{
{
HostPort: p,
},
}
exposedPorts[dockerPort] = struct{}{}
}
copyEnv := []string{
"ROUTER_EXTERNAL_HOST_HOSTNAME",
"ROUTER_EXTERNAL_HOST_USERNAME",
"ROUTER_EXTERNAL_HOST_PASSWORD",
"ROUTER_EXTERNAL_HOST_HTTP_VSERVER",
"ROUTER_EXTERNAL_HOST_HTTPS_VSERVER",
"ROUTER_EXTERNAL_HOST_INSECURE",
"ROUTER_EXTERNAL_HOST_PRIVKEY",
}
env := []string{
fmt.Sprintf("STATS_PORT=%d", routerStatsPort),
fmt.Sprintf("STATS_USERNAME=%s", statsUser),
fmt.Sprintf("STATS_PASSWORD=%s", statsPassword),
}
for _, name := range copyEnv {
val := os.Getenv(name)
if len(val) > 0 {
env = append(env, name+"="+val)
}
}
vols := ""
hostVols := []string{}
privkeyFilename := os.Getenv("ROUTER_EXTERNAL_HOST_PRIVKEY")
if len(privkeyFilename) != 0 {
vols = privkeyFilename
privkeyBindmount := fmt.Sprintf("%[1]s:%[1]s", privkeyFilename)
hostVols = append(hostVols, privkeyBindmount)
}
binary := os.Getenv("ROUTER_OPENSHIFT_BINARY")
if len(binary) != 0 {
hostVols = append(hostVols, fmt.Sprintf("%[1]s:/usr/bin/openshift", binary))
}
containerOpts := dockerClient.CreateContainerOptions{
Config: &dockerClient.Config{
Image: getRouterImage(),
Cmd: []string{"--master=" + masterIp, "--loglevel=4"},
Env: env,
ExposedPorts: exposedPorts,
VolumesFrom: vols,
},
HostConfig: &dockerClient.HostConfig{
Binds: hostVols,
},
}
container, err := dockerCli.CreateContainer(containerOpts)
if err != nil {
return "", err
}
dockerHostCfg := &dockerClient.HostConfig{NetworkMode: "host", PortBindings: portBindings}
err = dockerCli.StartContainer(container.ID, dockerHostCfg)
if err != nil {
return "", err
}
running := false
//wait for it to start
for i := 0; i < dockerRetries; i++ {
time.Sleep(time.Second * dockerWaitSeconds)
c, err := dockerCli.InspectContainer(container.ID)
if err != nil {
return "", err
}
if c.State.Running {
running = true
break
}
}
if !running {
return "", errors.New("Container did not start after 3 tries!")
}
return container.ID, nil
}
func(intstr *util.IntOrString, c fuzz.Continue) {
// util.IntOrString will panic if its kind is set wrong.
if c.RandBool() {
intstr.Kind = util.IntstrInt
intstr.IntVal = int(c.RandUint64())
intstr.StrVal = ""
} else {
intstr.Kind = util.IntstrString
intstr.IntVal = 0
intstr.StrVal = c.RandString()
}
},
func(pb map[docker.Port][]docker.PortBinding, c fuzz.Continue) {
// This is necessary because keys with nil values get omitted.
// TODO: Is this a bug?
pb[docker.Port(c.RandString())] = []docker.PortBinding{
{c.RandString(), c.RandString()},
{c.RandString(), c.RandString()},
}
},
func(pm map[string]docker.PortMapping, c fuzz.Continue) {
// This is necessary because keys with nil values get omitted.
// TODO: Is this a bug?
pm[c.RandString()] = docker.PortMapping{
c.RandString(): c.RandString(),
}
},
)
func TestLoad(t *testing.T) {
_, err := LoadSchemaForTest("v1beta1-swagger.json")
// createAndStartRouterContainer is responsible for deploying the router image in docker. It assumes that all router images
// will use a command line flag that can take --master which points to the master url
func createAndStartRouterContainer(dockerCli *dockerClient.Client, masterIp string, routerStatsPort int, reloadInterval int) (containerId string, err error) {
ports := []string{"80", "443"}
if routerStatsPort > 0 {
ports = append(ports, fmt.Sprintf("%d", routerStatsPort))
}
portBindings := make(map[dockerClient.Port][]dockerClient.PortBinding)
exposedPorts := map[dockerClient.Port]struct{}{}
for _, p := range ports {
dockerPort := dockerClient.Port(p + "/tcp")
portBindings[dockerPort] = []dockerClient.PortBinding{
{
HostPort: p,
},
}
exposedPorts[dockerPort] = struct{}{}
}
copyEnv := []string{
"ROUTER_EXTERNAL_HOST_HOSTNAME",
"ROUTER_EXTERNAL_HOST_USERNAME",
"ROUTER_EXTERNAL_HOST_PASSWORD",
"ROUTER_EXTERNAL_HOST_HTTP_VSERVER",
"ROUTER_EXTERNAL_HOST_HTTPS_VSERVER",
"ROUTER_EXTERNAL_HOST_INSECURE",
"ROUTER_EXTERNAL_HOST_PRIVKEY",
}
env := []string{
fmt.Sprintf("STATS_PORT=%d", routerStatsPort),
fmt.Sprintf("STATS_USERNAME=%s", statsUser),
fmt.Sprintf("STATS_PASSWORD=%s", statsPassword),
fmt.Sprintf("DEFAULT_CERTIFICATE=%s", defaultCert),
}
reloadIntVar := fmt.Sprintf("RELOAD_INTERVAL=%ds", reloadInterval)
env = append(env, reloadIntVar)
for _, name := range copyEnv {
val := os.Getenv(name)
if len(val) > 0 {
env = append(env, name+"="+val)
}
}
vols := ""
hostVols := []string{}
privkeyFilename := os.Getenv("ROUTER_EXTERNAL_HOST_PRIVKEY")
if len(privkeyFilename) != 0 {
vols = privkeyFilename
privkeyBindmount := fmt.Sprintf("%[1]s:%[1]s", privkeyFilename)
hostVols = append(hostVols, privkeyBindmount)
}
binary := os.Getenv("ROUTER_OPENSHIFT_BINARY")
if len(binary) != 0 {
hostVols = append(hostVols, fmt.Sprintf("%[1]s:/usr/bin/openshift", binary))
}
containerOpts := dockerClient.CreateContainerOptions{
Config: &dockerClient.Config{
Image: getRouterImage(),
Cmd: []string{"--master=" + masterIp, "--loglevel=4"},
Env: env,
ExposedPorts: exposedPorts,
VolumesFrom: vols,
},
HostConfig: &dockerClient.HostConfig{
Binds: hostVols,
},
}
container, err := dockerCli.CreateContainer(containerOpts)
if err != nil {
return "", err
}
dockerHostCfg := &dockerClient.HostConfig{NetworkMode: "host", PortBindings: portBindings}
err = dockerCli.StartContainer(container.ID, dockerHostCfg)
if err != nil {
return "", err
}
//wait for it to start
if err := wait.Poll(time.Millisecond*100, time.Second*30, func() (bool, error) {
c, err := dockerCli.InspectContainer(container.ID)
if err != nil {
return false, err
}
return c.State.Running, nil
}); err != nil {
return "", err
}
return container.ID, nil
}
// createContainer initializes a struct needed to call docker.client.CreateContainer()
func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task,
driverConfig *DockerDriverConfig, syslogAddr string) (docker.CreateContainerOptions, error) {
var c docker.CreateContainerOptions
if task.Resources == nil {
// Guard against missing resources. We should never have been able to
// schedule a job without specifying this.
d.logger.Println("[ERR] driver.docker: task.Resources is empty")
return c, fmt.Errorf("task.Resources is empty")
}
binds, err := d.containerBinds(ctx.AllocDir, task)
if err != nil {
return c, err
}
// Set environment variables.
d.taskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName))
d.taskEnv.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal))
config := &docker.Config{
Image: driverConfig.ImageName,
Hostname: driverConfig.Hostname,
User: task.User,
Tty: driverConfig.TTY,
OpenStdin: driverConfig.Interactive,
}
hostConfig := &docker.HostConfig{
// Convert MB to bytes. This is an absolute value.
Memory: int64(task.Resources.MemoryMB) * 1024 * 1024,
MemorySwap: -1,
// Convert Mhz to shares. This is a relative value.
CPUShares: int64(task.Resources.CPU),
// Binds are used to mount a host volume into the container. We mount a
// local directory for storage and a shared alloc directory that can be
// used to share data between different tasks in the same task group.
Binds: binds,
LogConfig: docker.LogConfig{
Type: "syslog",
Config: map[string]string{
"syslog-address": syslogAddr,
},
},
}
d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Config["image"])
d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Config["image"])
d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Config["image"])
// set privileged mode
hostPrivileged := d.config.ReadBoolDefault("docker.privileged.enabled", false)
if driverConfig.Privileged && !hostPrivileged {
return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent`)
}
hostConfig.Privileged = hostPrivileged
// set SHM size
if driverConfig.ShmSize != 0 {
hostConfig.ShmSize = driverConfig.ShmSize
}
// set DNS servers
for _, ip := range driverConfig.DNSServers {
if net.ParseIP(ip) != nil {
hostConfig.DNS = append(hostConfig.DNS, ip)
} else {
d.logger.Printf("[ERR] driver.docker: invalid ip address for container dns server: %s", ip)
}
}
// set DNS search domains
for _, domain := range driverConfig.DNSSearchDomains {
hostConfig.DNSSearch = append(hostConfig.DNSSearch, domain)
}
if driverConfig.IpcMode != "" {
if !hostPrivileged {
return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent, setting ipc mode not allowed`)
}
d.logger.Printf("[DEBUG] driver.docker: setting ipc mode to %s", driverConfig.IpcMode)
}
hostConfig.IpcMode = driverConfig.IpcMode
if driverConfig.PidMode != "" {
if !hostPrivileged {
return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent, setting pid mode not allowed`)
}
d.logger.Printf("[DEBUG] driver.docker: setting pid mode to %s", driverConfig.PidMode)
}
hostConfig.PidMode = driverConfig.PidMode
if driverConfig.UTSMode != "" {
if !hostPrivileged {
return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent, setting UTS mode not allowed`)
}
d.logger.Printf("[DEBUG] driver.docker: setting UTS mode to %s", driverConfig.UTSMode)
}
hostConfig.UTSMode = driverConfig.UTSMode
hostConfig.NetworkMode = driverConfig.NetworkMode
if hostConfig.NetworkMode == "" {
// docker default
d.logger.Println("[DEBUG] driver.docker: networking mode not specified; defaulting to bridge")
hostConfig.NetworkMode = "bridge"
}
// Setup port mapping and exposed ports
if len(task.Resources.Networks) == 0 {
d.logger.Println("[DEBUG] driver.docker: No network interfaces are available")
if len(driverConfig.PortMap) > 0 {
return c, fmt.Errorf("Trying to map ports but no network interface is available")
}
} else {
// TODO add support for more than one network
network := task.Resources.Networks[0]
publishedPorts := map[docker.Port][]docker.PortBinding{}
exposedPorts := map[docker.Port]struct{}{}
for _, port := range network.ReservedPorts {
// By default we will map the allocated port 1:1 to the container
containerPortInt := port.Value
// If the user has mapped a port using port_map we'll change it here
if mapped, ok := driverConfig.PortMap[port.Label]; ok {
containerPortInt = mapped
}
hostPortStr := strconv.Itoa(port.Value)
containerPort := docker.Port(strconv.Itoa(containerPortInt))
publishedPorts[containerPort+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}}
publishedPorts[containerPort+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}}
d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)", network.IP, port.Value, port.Value)
exposedPorts[containerPort+"/tcp"] = struct{}{}
exposedPorts[containerPort+"/udp"] = struct{}{}
d.logger.Printf("[DEBUG] driver.docker: exposed port %d", port.Value)
}
for _, port := range network.DynamicPorts {
// By default we will map the allocated port 1:1 to the container
containerPortInt := port.Value
// If the user has mapped a port using port_map we'll change it here
if mapped, ok := driverConfig.PortMap[port.Label]; ok {
containerPortInt = mapped
}
hostPortStr := strconv.Itoa(port.Value)
containerPort := docker.Port(strconv.Itoa(containerPortInt))
publishedPorts[containerPort+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}}
publishedPorts[containerPort+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}}
d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (mapped)", network.IP, port.Value, containerPortInt)
exposedPorts[containerPort+"/tcp"] = struct{}{}
exposedPorts[containerPort+"/udp"] = struct{}{}
d.logger.Printf("[DEBUG] driver.docker: exposed port %s", containerPort)
}
d.taskEnv.SetPortMap(driverConfig.PortMap)
hostConfig.PortBindings = publishedPorts
config.ExposedPorts = exposedPorts
}
d.taskEnv.Build()
parsedArgs := d.taskEnv.ParseAndReplace(driverConfig.Args)
// If the user specified a custom command to run as their entrypoint, we'll
// inject it here.
if driverConfig.Command != "" {
// Validate command
if err := validateCommand(driverConfig.Command, "args"); err != nil {
return c, err
}
cmd := []string{driverConfig.Command}
if len(driverConfig.Args) != 0 {
cmd = append(cmd, parsedArgs...)
}
d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s", strings.Join(cmd, " "))
config.Cmd = cmd
} else if len(driverConfig.Args) != 0 {
config.Cmd = parsedArgs
}
if len(driverConfig.Labels) > 0 {
config.Labels = driverConfig.Labels
d.logger.Printf("[DEBUG] driver.docker: applied labels on the container: %+v", config.Labels)
}
config.Env = d.taskEnv.EnvList()
containerName := fmt.Sprintf("%s-%s", task.Name, ctx.AllocID)
d.logger.Printf("[DEBUG] driver.docker: setting container name to: %s", containerName)
return docker.CreateContainerOptions{
Name: containerName,
Config: config,
HostConfig: hostConfig,
}, nil
}
func TestMakePortsAndBindings(t *testing.T) {
ports := []kubecontainer.PortMapping{
{
ContainerPort: 80,
HostPort: 8080,
HostIP: "127.0.0.1",
},
{
ContainerPort: 443,
HostPort: 443,
Protocol: "tcp",
},
{
ContainerPort: 444,
HostPort: 444,
Protocol: "udp",
},
{
ContainerPort: 445,
HostPort: 445,
Protocol: "foobar",
},
}
exposedPorts, bindings := makePortsAndBindings(ports)
if len(ports) != len(exposedPorts) ||
len(ports) != len(bindings) {
t.Errorf("Unexpected ports and bindings, %#v %#v %#v", ports, exposedPorts, bindings)
}
for key, value := range bindings {
switch value[0].HostPort {
case "8080":
if !reflect.DeepEqual(docker.Port("80/tcp"), key) {
t.Errorf("Unexpected docker port: %#v", key)
}
if value[0].HostIP != "127.0.0.1" {
t.Errorf("Unexpected host IP: %s", value[0].HostIP)
}
case "443":
if !reflect.DeepEqual(docker.Port("443/tcp"), key) {
t.Errorf("Unexpected docker port: %#v", key)
}
if value[0].HostIP != "" {
t.Errorf("Unexpected host IP: %s", value[0].HostIP)
}
case "444":
if !reflect.DeepEqual(docker.Port("444/udp"), key) {
t.Errorf("Unexpected docker port: %#v", key)
}
if value[0].HostIP != "" {
t.Errorf("Unexpected host IP: %s", value[0].HostIP)
}
case "445":
if !reflect.DeepEqual(docker.Port("445/tcp"), key) {
t.Errorf("Unexpected docker port: %#v", key)
}
if value[0].HostIP != "" {
t.Errorf("Unexpected host IP: %s", value[0].HostIP)
}
}
}
}
func (this *DockerClientEng1) Run(unit *core.Unit, callbackFunc func(*core.Dockerd, int, ...interface{})) error {
hostConfig := &docker.HostConfig{}
config := &docker.Config{
Image: unit.Image,
}
createContainerOptions := &docker.CreateContainerOptions{
Name: unit.Name,
Config: config,
HostConfig: hostConfig,
}
containerCreateResponse := &types.ContainerCreateResponse{}
/*
解析Unit参数
Config 是 create 所需的参数
HostConfig 是 run 所需的参数
这是 go-dockerclient 所定义的
参考：https://github.com/Tonnu/go-dockerclient/blob/master/container.go
当前用单线程来和所有的dockerd交互，到时候要改成携程。
*/
for _, p := range unit.Parameteres {
switch p.Type {
case "v": //-v Volume
hostConfig.Binds = append(hostConfig.Binds, p.Value)
case "p": //-p EXPOSE
rePort := regexp.MustCompile(".+/.+")
// 127.0.0.1:80:8080
re3 := regexp.MustCompile("(.+):(.+):(.+)")
if re3.MatchString(p.Value) {
t := re3.FindStringSubmatch(p.Value)
portBinding := &docker.PortBinding{
HostIP: t[1],
HostPort: t[2],
}
var containerPort string
if rePort.MatchString(t[3]) {
containerPort = t[3]
} else {
containerPort = fmt.Sprintf("%s/tcp", t[3])
}
hostConfig.PortBindings = make(map[docker.Port][]docker.PortBinding)
hostConfig.PortBindings[docker.Port(containerPort)] = append(hostConfig.PortBindings[docker.Port(containerPort)], *portBinding)
break
}
//80:8080
re2 := regexp.MustCompile("(.+):(.+)")
if re2.MatchString(p.Value) {
t := re2.FindStringSubmatch(p.Value)
portBinding := &docker.PortBinding{
HostPort: t[1],
}
hostConfig.PortBindings = make(map[docker.Port][]docker.PortBinding)
hostConfig.PortBindings[docker.Port(t[2])] = append(hostConfig.PortBindings[docker.Port(t[2])], *portBinding)
break
}
re1 := regexp.MustCompile("(.+)")
if re1.MatchString(p.Value) {
t := re2.FindStringSubmatch(p.Value)
portBinding := &docker.PortBinding{}
hostConfig.PortBindings = make(map[docker.Port][]docker.PortBinding)
hostConfig.PortBindings[docker.Port(t[1])] = append(hostConfig.PortBindings[docker.Port(t[1])], *portBinding)
break
}
}
}
//强行从registry pull最新版本的image
pullImageOptions := docker.PullImageOptions{}
// registry.ws.com/cst05001/nginx:latest
reImage3 := regexp.MustCompile("^(.*\\.\\w+)(/.*):(.*)")
// /cst05001/nginx:latest
reImage2 := regexp.MustCompile("^(/.*):(.*)")
if reImage3.MatchString(unit.Image) {
result := reImage3.FindStringSubmatch(unit.Image)
pullImageOptions.Registry = result[1]
pullImageOptions.Repository = result[2]
pullImageOptions.Tag = result[3]
} else if reImage2.MatchString(unit.Image) {
result := reImage2.FindStringSubmatch(unit.Image)
pullImageOptions.Registry = "http://docker.io"
pullImageOptions.Repository = result[1]
pullImageOptions.Tag = result[2]
} else {
pullImageOptions.Registry = "http://docker.io"
pullImageOptions.Repository = unit.Image
pullImageOptions.Tag = "latest"
}
for ptrDockerd, ptrClient := range this.ClientMap {
dockerd := &(*ptrDockerd)
client := &(*ptrClient)
go func() {
//第二个参数支持registry身份认证，还没处理。
err := client.PullImage(pullImageOptions, docker.AuthConfiguration{})
if err != nil {
beego.Error("Pull image ", pullImageOptions.Registry, pullImageOptions.Repository,
pullImageOptions.Tag, " at ", dockerd.GetIP(), " failed: ", err)
return
}
beego.Debug("Pull image ", pullImageOptions.Registry,
pullImageOptions.Repository, pullImageOptions.Tag, " at ", dockerd.GetIP(), " successed.")
container, err := client.CreateContainer(*createContainerOptions)
if err != nil {
beego.Error("Create container at ", dockerd.GetIP(), " failed: ", err)
containerCreateResponse.Warnings = append(containerCreateResponse.Warnings, err.Error())
callbackFunc(dockerd, dockerdengine.STATUS_ON_CREATE_FAILED, unit)
return
}
containerCreateResponse.ID = container.ID
callbackFunc(dockerd, dockerdengine.STATUS_ON_CREATE_SUCCESSED, unit)
// start container
err = client.StartContainer(containerCreateResponse.ID, hostConfig)
if err != nil {
beego.Error("Start container at ", dockerd.GetIP(), " failed: ", err)
callbackFunc(dockerd, dockerdengine.STATUS_ON_RUN_FAILED, unit)
return
}
beego.Debug("StartContainer at ", dockerd.GetIP(), " successed")
callbackFunc(dockerd, dockerdengine.STATUS_ON_RUN_SUCCESSED, unit)
}()
}
return nil
}
// fuzzerFor can randomly populate api objects that are destined for version.
func fuzzerFor(t *testing.T, version string, src rand.Source) *fuzz.Fuzzer {
f := fuzz.New().NilChance(.5).NumElements(1, 1)
if src != nil {
f.RandSource(src)
}
f.Funcs(
func(j *runtime.PluginBase, c fuzz.Continue) {
// Do nothing; this struct has only a Kind field and it must stay blank in memory.
},
func(j *runtime.TypeMeta, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = ""
j.Kind = ""
},
func(j *api.TypeMeta, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = ""
j.Kind = ""
},
func(j *api.ObjectMeta, c fuzz.Continue) {
j.Name = c.RandString()
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.SelfLink = c.RandString()
var sec, nsec int64
c.Fuzz(&sec)
c.Fuzz(&nsec)
j.CreationTimestamp = util.Unix(sec, nsec).Rfc3339Copy()
},
func(j *api.ListMeta, c fuzz.Continue) {
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.SelfLink = c.RandString()
},
func(j *api.PodPhase, c fuzz.Continue) {
statuses := []api.PodPhase{api.PodPending, api.PodRunning, api.PodFailed, api.PodUnknown}
*j = statuses[c.Rand.Intn(len(statuses))]
},
func(j *api.ReplicationControllerSpec, c fuzz.Continue) {
// TemplateRef must be nil for round trip
c.Fuzz(&j.Template)
if j.Template == nil {
// TODO: v1beta1/2 can't round trip a nil template correctly, fix by having v1beta1/2
// conversion compare converted object to nil via DeepEqual
j.Template = &api.PodTemplateSpec{}
}
j.Template.ObjectMeta = api.ObjectMeta{Labels: j.Template.ObjectMeta.Labels}
j.Template.Spec.NodeSelector = nil
c.Fuzz(&j.Selector)
j.Replicas = int(c.RandUint64())
},
func(j *api.ReplicationControllerStatus, c fuzz.Continue) {
// only replicas round trips
j.Replicas = int(c.RandUint64())
},
func(j *api.List, c fuzz.Continue) {
c.Fuzz(&j.ListMeta)
c.Fuzz(&j.Items)
if j.Items == nil {
j.Items = []runtime.Object{}
}
},
func(j *runtime.Object, c fuzz.Continue) {
if c.RandBool() {
*j = &runtime.Unknown{
TypeMeta: runtime.TypeMeta{Kind: "Something", APIVersion: "unknown"},
RawJSON: []byte(`{"apiVersion":"unknown","kind":"Something","someKey":"someValue"}`),
}
} else {
types := []runtime.Object{&api.Pod{}, &api.ReplicationController{}}
t := types[c.Rand.Intn(len(types))]
c.Fuzz(t)
*j = t
}
},
func(intstr *util.IntOrString, c fuzz.Continue) {
// util.IntOrString will panic if its kind is set wrong.
if c.RandBool() {
intstr.Kind = util.IntstrInt
intstr.IntVal = int(c.RandUint64())
intstr.StrVal = ""
} else {
intstr.Kind = util.IntstrString
intstr.IntVal = 0
intstr.StrVal = c.RandString()
}
},
func(pb map[docker.Port][]docker.PortBinding, c fuzz.Continue) {
// This is necessary because keys with nil values get omitted.
// TODO: Is this a bug?
pb[docker.Port(c.RandString())] = []docker.PortBinding{
{c.RandString(), c.RandString()},
{c.RandString(), c.RandString()},
}
},
func(pm map[string]docker.PortMapping, c fuzz.Continue) {
// This is necessary because keys with nil values get omitted.
// TODO: Is this a bug?
pm[c.RandString()] = docker.PortMapping{
c.RandString(): c.RandString(),
}
},
func(q *resource.Quantity, c fuzz.Continue) {
// Real Quantity fuzz testing is done elsewhere;
// this limited subset of functionality survives
// round-tripping to v1beta1/2.
q.Amount = &inf.Dec{}
q.Format = resource.DecimalExponent
//q.Amount.SetScale(inf.Scale(-c.Intn(12)))
q.Amount.SetUnscaled(c.Int63n(1000))
},
)
return f
}
// createAndStartRouterContainer is responsible for deploying the router image in docker. It assumes that all router images
// will use a command line flag that can take --master which points to the master url
func createAndStartRouterContainer(dockerCli *dockerClient.Client, masterIp string) (containerId string, err error) {
ports := []string{"80", "443"}
portBindings := make(map[dockerClient.Port][]dockerClient.PortBinding)
exposedPorts := map[dockerClient.Port]struct{}{}
for _, p := range ports {
dockerPort := dockerClient.Port(p + "/tcp")
portBindings[dockerPort] = []dockerClient.PortBinding{
{
HostPort: p,
},
}
exposedPorts[dockerPort] = struct{}{}
}
containerOpts := dockerClient.CreateContainerOptions{
Config: &dockerClient.Config{
Image: getRouterImage(),
Cmd: []string{"--master=" + masterIp, "--loglevel=4"},
ExposedPorts: exposedPorts,
},
}
container, err := dockerCli.CreateContainer(containerOpts)
if err != nil {
return "", err
}
dockerHostCfg := &dockerClient.HostConfig{NetworkMode: "host", PortBindings: portBindings}
err = dockerCli.StartContainer(container.ID, dockerHostCfg)
if err != nil {
return "", err
}
running := false
//wait for it to start
for i := 0; i < dockerRetries; i++ {
c, err := dockerCli.InspectContainer(container.ID)
if err != nil {
return "", err
}
if c.State.Running {
running = true
break
}
time.Sleep(time.Second * dockerWaitSeconds)
}
if !running {
return "", errors.New("Container did not start after 3 tries!")
}
return container.ID, nil
}
// FuzzerFor can randomly populate api objects that are destined for version.
func FuzzerFor(t *testing.T, version string, src rand.Source) *fuzz.Fuzzer {
f := fuzz.New().NilChance(.5).NumElements(1, 1)
if src != nil {
f.RandSource(src)
}
f.Funcs(
func(j *int, c fuzz.Continue) {
*j = int(c.Int31())
},
func(j **int, c fuzz.Continue) {
if c.RandBool() {
i := int(c.Int31())
*j = &i
} else {
*j = nil
}
},
func(j *runtime.PluginBase, c fuzz.Continue) {
// Do nothing; this struct has only a Kind field and it must stay blank in memory.
},
func(j *runtime.TypeMeta, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = ""
j.Kind = ""
},
func(j *unversioned.TypeMeta, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = ""
j.Kind = ""
},
func(j *api.ObjectMeta, c fuzz.Continue) {
j.Name = c.RandString()
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.SelfLink = c.RandString()
j.UID = types.UID(c.RandString())
j.GenerateName = c.RandString()
var sec, nsec int64
c.Fuzz(&sec)
c.Fuzz(&nsec)
j.CreationTimestamp = unversioned.Unix(sec, nsec).Rfc3339Copy()
},
func(j *api.ObjectReference, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = c.RandString()
j.Kind = c.RandString()
j.Namespace = c.RandString()
j.Name = c.RandString()
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.FieldPath = c.RandString()
},
func(j *unversioned.ListMeta, c fuzz.Continue) {
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.SelfLink = c.RandString()
},
func(j *unversioned.ListOptions, c fuzz.Continue) {
// TODO: add some parsing
label, _ := labels.Parse("a=b")
j.LabelSelector = unversioned.LabelSelector{label}
field, _ := fields.ParseSelector("a=b")
j.FieldSelector = unversioned.FieldSelector{field}
},
func(j *api.PodExecOptions, c fuzz.Continue) {
j.Stdout = true
j.Stderr = true
},
func(j *api.PodAttachOptions, c fuzz.Continue) {
j.Stdout = true
j.Stderr = true
},
func(s *api.PodSpec, c fuzz.Continue) {
c.FuzzNoCustom(s)
// has a default value
ttl := int64(30)
if c.RandBool() {
ttl = int64(c.Uint32())
}
s.TerminationGracePeriodSeconds = &ttl
c.Fuzz(s.SecurityContext)
if s.SecurityContext == nil {
s.SecurityContext = new(api.PodSecurityContext)
}
},
func(j *api.PodPhase, c fuzz.Continue) {
statuses := []api.PodPhase{api.PodPending, api.PodRunning, api.PodFailed, api.PodUnknown}
*j = statuses[c.Rand.Intn(len(statuses))]
},
func(j *api.Binding, c fuzz.Continue) {
c.Fuzz(&j.ObjectMeta)
j.Target.Name = c.RandString()
},
func(j *api.ReplicationControllerSpec, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
//j.TemplateRef = nil // this is required for round trip
},
func(j *extensions.DeploymentStrategy, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
// Ensure that strategyType is one of valid values.
strategyTypes := []extensions.DeploymentStrategyType{extensions.RecreateDeploymentStrategyType, extensions.RollingUpdateDeploymentStrategyType}
j.Type = strategyTypes[c.Rand.Intn(len(strategyTypes))]
if j.Type != extensions.RollingUpdateDeploymentStrategyType {
j.RollingUpdate = nil
} else {
rollingUpdate := extensions.RollingUpdateDeployment{}
if c.RandBool() {
rollingUpdate.MaxUnavailable = intstr.FromInt(int(c.RandUint64()))
rollingUpdate.MaxSurge = intstr.FromInt(int(c.RandUint64()))
} else {
rollingUpdate.MaxSurge = intstr.FromString(fmt.Sprintf("%d%%", c.RandUint64()))
}
j.RollingUpdate = &rollingUpdate
}
},
func(j *extensions.JobSpec, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
completions := int(c.Rand.Int31())
parallelism := int(c.Rand.Int31())
j.Completions = &completions
j.Parallelism = ¶llelism
},
func(j *api.List, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
// TODO: uncomment when round trip starts from a versioned object
if false { //j.Items == nil {
j.Items = []runtime.Object{}
}
},
func(j *runtime.Object, c fuzz.Continue) {
// TODO: uncomment when round trip starts from a versioned object
if true { //c.RandBool() {
*j = &runtime.Unknown{
// apiVersion has rules now. Since it includes <group>/<version> and only `v1` can be bare,
// then this must choose a valid format to deserialize
TypeMeta: runtime.TypeMeta{Kind: "Something", APIVersion: "unknown.group/unknown"},
RawJSON: []byte(`{"apiVersion":"unknown.group/unknown","kind":"Something","someKey":"someValue"}`),
}
} else {
types := []runtime.Object{&api.Pod{}, &api.ReplicationController{}}
t := types[c.Rand.Intn(len(types))]
c.Fuzz(t)
*j = t
}
},
func(pb map[docker.Port][]docker.PortBinding, c fuzz.Continue) {
// This is necessary because keys with nil values get omitted.
// TODO: Is this a bug?
pb[docker.Port(c.RandString())] = []docker.PortBinding{
{c.RandString(), c.RandString()},
{c.RandString(), c.RandString()},
}
},
func(pm map[string]docker.PortMapping, c fuzz.Continue) {
// This is necessary because keys with nil values get omitted.
// TODO: Is this a bug?
pm[c.RandString()] = docker.PortMapping{
c.RandString(): c.RandString(),
}
},
func(q *api.ResourceRequirements, c fuzz.Continue) {
randomQuantity := func() resource.Quantity {
return *resource.NewQuantity(c.Int63n(1000), resource.DecimalExponent)
}
q.Limits = make(api.ResourceList)
q.Requests = make(api.ResourceList)
cpuLimit := randomQuantity()
q.Limits[api.ResourceCPU] = *cpuLimit.Copy()
q.Requests[api.ResourceCPU] = *cpuLimit.Copy()
memoryLimit := randomQuantity()
q.Limits[api.ResourceMemory] = *memoryLimit.Copy()
q.Requests[api.ResourceMemory] = *memoryLimit.Copy()
storageLimit := randomQuantity()
q.Limits[api.ResourceStorage] = *storageLimit.Copy()
q.Requests[api.ResourceStorage] = *storageLimit.Copy()
},
func(q *api.LimitRangeItem, c fuzz.Continue) {
randomQuantity := func() resource.Quantity {
return *resource.NewQuantity(c.Int63n(1000), resource.DecimalExponent)
}
cpuLimit := randomQuantity()
q.Type = api.LimitTypeContainer
q.Default = make(api.ResourceList)
q.Default[api.ResourceCPU] = *(cpuLimit.Copy())
q.DefaultRequest = make(api.ResourceList)
q.DefaultRequest[api.ResourceCPU] = *(cpuLimit.Copy())
q.Max = make(api.ResourceList)
q.Max[api.ResourceCPU] = *(cpuLimit.Copy())
q.Min = make(api.ResourceList)
q.Min[api.ResourceCPU] = *(cpuLimit.Copy())
q.MaxLimitRequestRatio = make(api.ResourceList)
q.MaxLimitRequestRatio[api.ResourceCPU] = resource.MustParse("10")
},
func(p *api.PullPolicy, c fuzz.Continue) {
policies := []api.PullPolicy{api.PullAlways, api.PullNever, api.PullIfNotPresent}
*p = policies[c.Rand.Intn(len(policies))]
},
func(rp *api.RestartPolicy, c fuzz.Continue) {
policies := []api.RestartPolicy{api.RestartPolicyAlways, api.RestartPolicyNever, api.RestartPolicyOnFailure}
*rp = policies[c.Rand.Intn(len(policies))]
},
// Only api.DownwardAPIVolumeFile needs to have a specific func since FieldRef has to be
// defaulted to a version otherwise roundtrip will fail
// For the remaining volume plugins the default fuzzer is enough.
func(m *api.DownwardAPIVolumeFile, c fuzz.Continue) {
m.Path = c.RandString()
versions := []string{"v1"}
m.FieldRef.APIVersion = versions[c.Rand.Intn(len(versions))]
m.FieldRef.FieldPath = c.RandString()
},
func(vs *api.VolumeSource, c fuzz.Continue) {
// Exactly one of the fields must be set.
v := reflect.ValueOf(vs).Elem()
i := int(c.RandUint64() % uint64(v.NumField()))
t := v.Field(i).Addr()
for v.Field(i).IsNil() {
c.Fuzz(t.Interface())
}
},
func(d *api.DNSPolicy, c fuzz.Continue) {
policies := []api.DNSPolicy{api.DNSClusterFirst, api.DNSDefault}
*d = policies[c.Rand.Intn(len(policies))]
},
func(p *api.Protocol, c fuzz.Continue) {
protocols := []api.Protocol{api.ProtocolTCP, api.ProtocolUDP}
*p = protocols[c.Rand.Intn(len(protocols))]
},
func(p *api.ServiceAffinity, c fuzz.Continue) {
types := []api.ServiceAffinity{api.ServiceAffinityClientIP, api.ServiceAffinityNone}
*p = types[c.Rand.Intn(len(types))]
},
func(p *api.ServiceType, c fuzz.Continue) {
types := []api.ServiceType{api.ServiceTypeClusterIP, api.ServiceTypeNodePort, api.ServiceTypeLoadBalancer}
*p = types[c.Rand.Intn(len(types))]
},
func(ct *api.Container, c fuzz.Continue) {
c.FuzzNoCustom(ct) // fuzz self without calling this function again
ct.TerminationMessagePath = "/" + ct.TerminationMessagePath // Must be non-empty
},
func(p *api.Probe, c fuzz.Continue) {
c.FuzzNoCustom(p)
// These fields have default values.
intFieldsWithDefaults := [...]string{"TimeoutSeconds", "PeriodSeconds", "SuccessThreshold", "FailureThreshold"}
v := reflect.ValueOf(p).Elem()
for _, field := range intFieldsWithDefaults {
f := v.FieldByName(field)
if f.Int() == 0 {
f.SetInt(1)
}
}
},
func(ev *api.EnvVar, c fuzz.Continue) {
ev.Name = c.RandString()
if c.RandBool() {
ev.Value = c.RandString()
} else {
ev.ValueFrom = &api.EnvVarSource{}
ev.ValueFrom.FieldRef = &api.ObjectFieldSelector{}
versions := registered.RegisteredGroupVersions
ev.ValueFrom.FieldRef.APIVersion = versions[c.Rand.Intn(len(versions))].String()
ev.ValueFrom.FieldRef.FieldPath = c.RandString()
}
},
func(sc *api.SecurityContext, c fuzz.Continue) {
c.FuzzNoCustom(sc) // fuzz self without calling this function again
if c.RandBool() {
priv := c.RandBool()
sc.Privileged = &priv
}
if c.RandBool() {
sc.Capabilities = &api.Capabilities{
Add: make([]api.Capability, 0),
Drop: make([]api.Capability, 0),
}
c.Fuzz(&sc.Capabilities.Add)
c.Fuzz(&sc.Capabilities.Drop)
}
},
func(s *api.Secret, c fuzz.Continue) {
c.FuzzNoCustom(s) // fuzz self without calling this function again
s.Type = api.SecretTypeOpaque
},
func(pv *api.PersistentVolume, c fuzz.Continue) {
c.FuzzNoCustom(pv) // fuzz self without calling this function again
types := []api.PersistentVolumePhase{api.VolumeAvailable, api.VolumePending, api.VolumeBound, api.VolumeReleased, api.VolumeFailed}
pv.Status.Phase = types[c.Rand.Intn(len(types))]
pv.Status.Message = c.RandString()
reclamationPolicies := []api.PersistentVolumeReclaimPolicy{api.PersistentVolumeReclaimRecycle, api.PersistentVolumeReclaimRetain}
pv.Spec.PersistentVolumeReclaimPolicy = reclamationPolicies[c.Rand.Intn(len(reclamationPolicies))]
},
func(pvc *api.PersistentVolumeClaim, c fuzz.Continue) {
c.FuzzNoCustom(pvc) // fuzz self without calling this function again
types := []api.PersistentVolumeClaimPhase{api.ClaimBound, api.ClaimPending}
pvc.Status.Phase = types[c.Rand.Intn(len(types))]
},
func(s *api.NamespaceSpec, c fuzz.Continue) {
s.Finalizers = []api.FinalizerName{api.FinalizerKubernetes}
},
func(s *api.NamespaceStatus, c fuzz.Continue) {
s.Phase = api.NamespaceActive
},
func(http *api.HTTPGetAction, c fuzz.Continue) {
c.FuzzNoCustom(http) // fuzz self without calling this function again
http.Path = "/" + http.Path // can't be blank
http.Scheme = "x" + http.Scheme // can't be blank
},
func(ss *api.ServiceSpec, c fuzz.Continue) {
c.FuzzNoCustom(ss) // fuzz self without calling this function again
if len(ss.Ports) == 0 {
// There must be at least 1 port.
ss.Ports = append(ss.Ports, api.ServicePort{})
c.Fuzz(&ss.Ports[0])
}
for i := range ss.Ports {
switch ss.Ports[i].TargetPort.Type {
case intstr.Int:
ss.Ports[i].TargetPort.IntVal = 1 + ss.Ports[i].TargetPort.IntVal%65535 // non-zero
case intstr.String:
ss.Ports[i].TargetPort.StrVal = "x" + ss.Ports[i].TargetPort.StrVal // non-empty
}
}
},
func(n *api.Node, c fuzz.Continue) {
c.FuzzNoCustom(n)
n.Spec.ExternalID = "external"
},
func(s *extensions.APIVersion, c fuzz.Continue) {
// We can't use c.RandString() here because it may generate empty
// string, which will cause tests failure.
s.APIGroup = "something"
},
func(s *extensions.HorizontalPodAutoscalerSpec, c fuzz.Continue) {
c.FuzzNoCustom(s) // fuzz self without calling this function again
minReplicas := int(c.Rand.Int31())
s.MinReplicas = &minReplicas
s.CPUUtilization = &extensions.CPUTargetUtilization{TargetPercentage: int(int32(c.RandUint64()))}
},
)
return f
}
// FuzzerFor can randomly populate api objects that are destined for version.
func FuzzerFor(t *testing.T, version string, src rand.Source) *fuzz.Fuzzer {
f := fuzz.New().NilChance(.5).NumElements(1, 1)
if src != nil {
f.RandSource(src)
}
f.Funcs(
func(j *runtime.PluginBase, c fuzz.Continue) {
// Do nothing; this struct has only a Kind field and it must stay blank in memory.
},
func(j *runtime.TypeMeta, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = ""
j.Kind = ""
},
func(j *api.TypeMeta, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = ""
j.Kind = ""
},
func(j *api.ObjectMeta, c fuzz.Continue) {
j.Name = c.RandString()
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.SelfLink = c.RandString()
j.UID = types.UID(c.RandString())
j.GenerateName = c.RandString()
var sec, nsec int64
c.Fuzz(&sec)
c.Fuzz(&nsec)
j.CreationTimestamp = util.Unix(sec, nsec).Rfc3339Copy()
},
func(j *api.ObjectReference, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = c.RandString()
j.Kind = c.RandString()
j.Namespace = c.RandString()
j.Name = c.RandString()
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.FieldPath = c.RandString()
},
func(j *api.ListMeta, c fuzz.Continue) {
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.SelfLink = c.RandString()
},
func(j *api.ListOptions, c fuzz.Continue) {
// TODO: add some parsing
j.LabelSelector, _ = labels.Parse("a=b")
j.FieldSelector, _ = fields.ParseSelector("a=b")
},
func(j *api.PodSpec, c fuzz.Continue) {
c.FuzzNoCustom(j)
// has a default value
ttl := int64(30)
if c.RandBool() {
ttl = int64(c.Uint32())
}
j.TerminationGracePeriodSeconds = &ttl
},
func(j *api.PodPhase, c fuzz.Continue) {
statuses := []api.PodPhase{api.PodPending, api.PodRunning, api.PodFailed, api.PodUnknown}
*j = statuses[c.Rand.Intn(len(statuses))]
},
func(j *api.PodTemplateSpec, c fuzz.Continue) {
// TODO: v1beta1/2 can't round trip a nil template correctly, fix by having v1beta1/2
// conversion compare converted object to nil via DeepEqual
j.ObjectMeta = api.ObjectMeta{}
c.Fuzz(&j.ObjectMeta)
j.ObjectMeta = api.ObjectMeta{Labels: j.ObjectMeta.Labels}
j.Spec = api.PodSpec{}
c.Fuzz(&j.Spec)
},
func(j *api.Binding, c fuzz.Continue) {
c.Fuzz(&j.ObjectMeta)
j.Target.Name = c.RandString()
},
func(j *api.ReplicationControllerSpec, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
//j.TemplateRef = nil // this is required for round trip
},
func(j *experimental.DeploymentStrategy, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
// Ensure that strategyType is one of valid values.
strategyTypes := []experimental.DeploymentType{experimental.DeploymentRecreate, experimental.DeploymentRollingUpdate}
j.Type = strategyTypes[c.Rand.Intn(len(strategyTypes))]
if j.Type != experimental.DeploymentRollingUpdate {
j.RollingUpdate = nil
} else {
rollingUpdate := experimental.RollingUpdateDeployment{}
if c.RandBool() {
rollingUpdate.MaxUnavailable = util.NewIntOrStringFromInt(int(c.RandUint64()))
rollingUpdate.MaxSurge = util.NewIntOrStringFromInt(int(c.RandUint64()))
} else {
rollingUpdate.MaxSurge = util.NewIntOrStringFromString(fmt.Sprintf("%d%%", c.RandUint64()))
}
j.RollingUpdate = &rollingUpdate
}
},
func(j *api.List, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
// TODO: uncomment when round trip starts from a versioned object
if false { //j.Items == nil {
j.Items = []runtime.Object{}
}
},
func(j *runtime.Object, c fuzz.Continue) {
// TODO: uncomment when round trip starts from a versioned object
if true { //c.RandBool() {
*j = &runtime.Unknown{
TypeMeta: runtime.TypeMeta{Kind: "Something", APIVersion: "unknown"},
RawJSON: []byte(`{"apiVersion":"unknown","kind":"Something","someKey":"someValue"}`),
}
} else {
types := []runtime.Object{&api.Pod{}, &api.ReplicationController{}}
t := types[c.Rand.Intn(len(types))]
c.Fuzz(t)
*j = t
}
},
func(pb map[docker.Port][]docker.PortBinding, c fuzz.Continue) {
// This is necessary because keys with nil values get omitted.
// TODO: Is this a bug?
pb[docker.Port(c.RandString())] = []docker.PortBinding{
{c.RandString(), c.RandString()},
{c.RandString(), c.RandString()},
}
},
func(pm map[string]docker.PortMapping, c fuzz.Continue) {
// This is necessary because keys with nil values get omitted.
// TODO: Is this a bug?
pm[c.RandString()] = docker.PortMapping{
c.RandString(): c.RandString(),
}
},
func(q *resource.Quantity, c fuzz.Continue) {
// Real Quantity fuzz testing is done elsewhere;
// this limited subset of functionality survives
// round-tripping to v1beta1/2.
q.Amount = &inf.Dec{}
q.Format = resource.DecimalExponent
//q.Amount.SetScale(inf.Scale(-c.Intn(12)))
q.Amount.SetUnscaled(c.Int63n(1000))
},
func(q *api.ResourceRequirements, c fuzz.Continue) {
randomQuantity := func() resource.Quantity {
return *resource.NewQuantity(c.Int63n(1000), resource.DecimalExponent)
}
q.Limits = make(api.ResourceList)
q.Requests = make(api.ResourceList)
cpuLimit := randomQuantity()
q.Limits[api.ResourceCPU] = *cpuLimit.Copy()
q.Requests[api.ResourceCPU] = *cpuLimit.Copy()
memoryLimit := randomQuantity()
q.Limits[api.ResourceMemory] = *memoryLimit.Copy()
q.Requests[api.ResourceMemory] = *memoryLimit.Copy()
storageLimit := randomQuantity()
q.Limits[api.ResourceStorage] = *storageLimit.Copy()
q.Requests[api.ResourceStorage] = *storageLimit.Copy()
},
func(q *api.LimitRangeItem, c fuzz.Continue) {
randomQuantity := func() resource.Quantity {
return *resource.NewQuantity(c.Int63n(1000), resource.DecimalExponent)
}
cpuLimit := randomQuantity()
q.Type = api.LimitTypeContainer
q.Default = make(api.ResourceList)
q.Default[api.ResourceCPU] = *(cpuLimit.Copy())
q.DefaultRequest = make(api.ResourceList)
q.DefaultRequest[api.ResourceCPU] = *(cpuLimit.Copy())
q.Max = make(api.ResourceList)
q.Max[api.ResourceCPU] = *(cpuLimit.Copy())
q.Min = make(api.ResourceList)
q.Min[api.ResourceCPU] = *(cpuLimit.Copy())
q.MaxLimitRequestRatio = make(api.ResourceList)
q.MaxLimitRequestRatio[api.ResourceCPU] = resource.MustParse("10")
},
func(p *api.PullPolicy, c fuzz.Continue) {
policies := []api.PullPolicy{api.PullAlways, api.PullNever, api.PullIfNotPresent}
*p = policies[c.Rand.Intn(len(policies))]
},
func(rp *api.RestartPolicy, c fuzz.Continue) {
policies := []api.RestartPolicy{api.RestartPolicyAlways, api.RestartPolicyNever, api.RestartPolicyOnFailure}
*rp = policies[c.Rand.Intn(len(policies))]
},
func(vs *api.VolumeSource, c fuzz.Continue) {
// Exactly one of the fields must be set.
v := reflect.ValueOf(vs).Elem()
i := int(c.RandUint64() % uint64(v.NumField()))
v = v.Field(i).Addr()
// Use a new fuzzer which cannot populate nil to ensure one field will be set.
f := fuzz.New().NilChance(0).NumElements(1, 1)
f.Funcs(
// Only api.DownwardAPIVolumeFile needs to have a specific func since FieldRef has to be
// defaulted to a version otherwise roundtrip will fail
// For the remaining volume plugins the default fuzzer is enough.
func(m *api.DownwardAPIVolumeFile, c fuzz.Continue) {
m.Path = c.RandString()
versions := []string{"v1"}
m.FieldRef.APIVersion = versions[c.Rand.Intn(len(versions))]
m.FieldRef.FieldPath = c.RandString()
},
).Fuzz(v.Interface())
},
func(d *api.DNSPolicy, c fuzz.Continue) {
policies := []api.DNSPolicy{api.DNSClusterFirst, api.DNSDefault}
*d = policies[c.Rand.Intn(len(policies))]
},
func(p *api.Protocol, c fuzz.Continue) {
protocols := []api.Protocol{api.ProtocolTCP, api.ProtocolUDP}
*p = protocols[c.Rand.Intn(len(protocols))]
},
func(p *api.ServiceAffinity, c fuzz.Continue) {
types := []api.ServiceAffinity{api.ServiceAffinityClientIP, api.ServiceAffinityNone}
*p = types[c.Rand.Intn(len(types))]
},
func(p *api.ServiceType, c fuzz.Continue) {
types := []api.ServiceType{api.ServiceTypeClusterIP, api.ServiceTypeNodePort, api.ServiceTypeLoadBalancer}
*p = types[c.Rand.Intn(len(types))]
},
func(ct *api.Container, c fuzz.Continue) {
c.FuzzNoCustom(ct) // fuzz self without calling this function again
ct.TerminationMessagePath = "/" + ct.TerminationMessagePath // Must be non-empty
},
func(ev *api.EnvVar, c fuzz.Continue) {
ev.Name = c.RandString()
if c.RandBool() {
ev.Value = c.RandString()
} else {
ev.ValueFrom = &api.EnvVarSource{}
ev.ValueFrom.FieldRef = &api.ObjectFieldSelector{}
versions := registered.RegisteredVersions
ev.ValueFrom.FieldRef.APIVersion = versions[c.Rand.Intn(len(versions))]
ev.ValueFrom.FieldRef.FieldPath = c.RandString()
}
},
func(sc *api.SecurityContext, c fuzz.Continue) {
c.FuzzNoCustom(sc) // fuzz self without calling this function again
priv := c.RandBool()
sc.Privileged = &priv
sc.Capabilities = &api.Capabilities{
Add: make([]api.Capability, 0),
Drop: make([]api.Capability, 0),
}
c.Fuzz(&sc.Capabilities.Add)
c.Fuzz(&sc.Capabilities.Drop)
},
func(e *api.Event, c fuzz.Continue) {
c.FuzzNoCustom(e) // fuzz self without calling this function again
// Fix event count to 1, otherwise, if a v1beta1 or v1beta2 event has a count set arbitrarily, it's count is ignored
if e.FirstTimestamp.IsZero() {
e.Count = 1
} else {
c.Fuzz(&e.Count)
}
},
func(s *api.Secret, c fuzz.Continue) {
c.FuzzNoCustom(s) // fuzz self without calling this function again
s.Type = api.SecretTypeOpaque
},
func(pv *api.PersistentVolume, c fuzz.Continue) {
c.FuzzNoCustom(pv) // fuzz self without calling this function again
types := []api.PersistentVolumePhase{api.VolumeAvailable, api.VolumePending, api.VolumeBound, api.VolumeReleased, api.VolumeFailed}
pv.Status.Phase = types[c.Rand.Intn(len(types))]
pv.Status.Message = c.RandString()
reclamationPolicies := []api.PersistentVolumeReclaimPolicy{api.PersistentVolumeReclaimRecycle, api.PersistentVolumeReclaimRetain}
pv.Spec.PersistentVolumeReclaimPolicy = reclamationPolicies[c.Rand.Intn(len(reclamationPolicies))]
},
func(pvc *api.PersistentVolumeClaim, c fuzz.Continue) {
c.FuzzNoCustom(pvc) // fuzz self without calling this function again
types := []api.PersistentVolumeClaimPhase{api.ClaimBound, api.ClaimPending}
pvc.Status.Phase = types[c.Rand.Intn(len(types))]
},
func(s *api.NamespaceSpec, c fuzz.Continue) {
s.Finalizers = []api.FinalizerName{api.FinalizerKubernetes}
},
func(s *api.NamespaceStatus, c fuzz.Continue) {
s.Phase = api.NamespaceActive
},
func(http *api.HTTPGetAction, c fuzz.Continue) {
c.FuzzNoCustom(http) // fuzz self without calling this function again
http.Path = "/" + http.Path // can't be blank
http.Scheme = "x" + http.Scheme // can't be blank
},
func(ss *api.ServiceSpec, c fuzz.Continue) {
c.FuzzNoCustom(ss) // fuzz self without calling this function again
if len(ss.Ports) == 0 {
// There must be at least 1 port.
ss.Ports = append(ss.Ports, api.ServicePort{})
c.Fuzz(&ss.Ports[0])
}
for i := range ss.Ports {
switch ss.Ports[i].TargetPort.Kind {
case util.IntstrInt:
ss.Ports[i].TargetPort.IntVal = 1 + ss.Ports[i].TargetPort.IntVal%65535 // non-zero
case util.IntstrString:
ss.Ports[i].TargetPort.StrVal = "x" + ss.Ports[i].TargetPort.StrVal // non-empty
}
}
},
func(n *api.Node, c fuzz.Continue) {
c.FuzzNoCustom(n)
n.Spec.ExternalID = "external"
},
func(s *experimental.APIVersion, c fuzz.Continue) {
// We can't use c.RandString() here because it may generate empty
// string, which will cause tests failure.
s.APIGroup = "something"
},
)
return f
}
// createContainer initializes a struct needed to call docker.client.CreateContainer()
func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task) (docker.CreateContainerOptions, error) {
var c docker.CreateContainerOptions
if task.Resources == nil {
d.logger.Printf("[ERR] driver.docker: task.Resources is empty")
return c, fmt.Errorf("task.Resources is nil and we can't constrain resource usage. We shouldn't have been able to schedule this in the first place.")
}
binds, err := d.containerBinds(ctx.AllocDir, task)
if err != nil {
return c, err
}
hostConfig := &docker.HostConfig{
// Convert MB to bytes. This is an absolute value.
//
// This value represents the total amount of memory a process can use.
// Swap is added to total memory and is managed by the OS, not docker.
// Since this may cause other processes to swap and cause system
// instability, we will simply not use swap.
//
// See: https://www.kernel.org/doc/Documentation/cgroups/memory.txt
Memory: int64(task.Resources.MemoryMB) * 1024 * 1024,
MemorySwap: -1,
// Convert Mhz to shares. This is a relative value.
//
// There are two types of CPU limiters available: Shares and Quotas. A
// Share allows a particular process to have a proportion of CPU time
// relative to other processes; 1024 by default. A CPU Quota is enforced
// over a Period of time and is a HARD limit on the amount of CPU time a
// process can use. Processes with quotas cannot burst, while processes
// with shares can, so we'll use shares.
//
// The simplest scale is 1 share to 1 MHz so 1024 = 1GHz. This means any
// given process will have at least that amount of resources, but likely
// more since it is (probably) rare that the machine will run at 100%
// CPU. This scale will cease to work if a node is overprovisioned.
//
// See:
// - https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
// - https://www.kernel.org/doc/Documentation/scheduler/sched-design-CFS.txt
CPUShares: int64(task.Resources.CPU),
// Binds are used to mount a host volume into the container. We mount a
// local directory for storage and a shared alloc directory that can be
// used to share data between different tasks in the same task group.
Binds: binds,
}
d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Config["image"])
d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Config["image"])
d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Config["image"])
// set privileged mode
hostPrivileged, err := strconv.ParseBool(d.config.ReadDefault("docker.privileged.enabled", "false"))
if err != nil {
return c, fmt.Errorf("Unable to parse docker.privileged.enabled: %s", err)
}
if v, ok := task.Config["privileged"]; ok {
taskPrivileged, err := strconv.ParseBool(v)
if err != nil {
return c, fmt.Errorf("Unable to parse boolean value from task config option 'privileged': %v", err)
}
if taskPrivileged && !hostPrivileged {
return c, fmt.Errorf(`Unable to set privileged flag since "docker.privileged.enabled" is false`)
}
hostConfig.Privileged = taskPrivileged
}
// set DNS servers
dns, ok := task.Config["dns-servers"]
if ok && dns != "" {
for _, v := range strings.Split(dns, ",") {
ip := strings.TrimSpace(v)
if net.ParseIP(ip) != nil {
hostConfig.DNS = append(hostConfig.DNS, ip)
} else {
d.logger.Printf("[ERR] driver.docker: invalid ip address for container dns server: %s", ip)
}
}
}
// set DNS search domains
dnsSearch, ok := task.Config["search-domains"]
if ok && dnsSearch != "" {
for _, v := range strings.Split(dnsSearch, ",") {
hostConfig.DNSSearch = append(hostConfig.DNSSearch, strings.TrimSpace(v))
}
}
mode, ok := task.Config["network_mode"]
if !ok || mode == "" {
// docker default
d.logger.Printf("[WARN] driver.docker: no mode specified for networking, defaulting to bridge")
mode = "bridge"
}
// Ignore the container mode for now
switch mode {
case "default", "bridge", "none", "host":
d.logger.Printf("[DEBUG] driver.docker: using %s as network mode", mode)
default:
d.logger.Printf("[ERR] driver.docker: invalid setting for network mode: %s", mode)
return c, fmt.Errorf("Invalid setting for network mode: %s", mode)
}
hostConfig.NetworkMode = mode
// Setup port mapping (equivalent to -p on docker CLI). Ports must already be
// exposed in the container.
if len(task.Resources.Networks) == 0 {
d.logger.Print("[WARN] driver.docker: No networks are available for port mapping")
} else {
network := task.Resources.Networks[0]
dockerPorts := map[docker.Port][]docker.PortBinding{}
for _, port := range network.ListStaticPorts() {
dockerPorts[docker.Port(strconv.Itoa(port)+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}}
dockerPorts[docker.Port(strconv.Itoa(port)+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}}
d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)\n", network.IP, port, port)
}
for label, port := range network.MapDynamicPorts() {
// If the label is numeric we expect that there is a service
// listening on that port inside the container. In this case we'll
// setup a mapping from our random host port to the label port.
//
// Otherwise we'll setup a direct 1:1 mapping from the host port to
// the container, and assume that the process inside will read the
// environment variable and bind to the correct port.
if _, err := strconv.Atoi(label); err == nil {
dockerPorts[docker.Port(label+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}}
dockerPorts[docker.Port(label+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}}
d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %s (mapped)", network.IP, port, label)
} else {
dockerPorts[docker.Port(strconv.Itoa(port)+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}}
dockerPorts[docker.Port(strconv.Itoa(port)+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}}
d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d for label %s\n", network.IP, port, port, label)
}
}
hostConfig.PortBindings = dockerPorts
}
// Create environment variables.
env := TaskEnvironmentVariables(ctx, task)
env.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName))
env.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal))
config := &docker.Config{
Env: env.List(),
Image: task.Config["image"],
}
rawArgs, hasArgs := task.Config["args"]
parsedArgs, err := args.ParseAndReplace(rawArgs, env.Map())
if err != nil {
return c, err
}
// If the user specified a custom command to run, we'll inject it here.
if command, ok := task.Config["command"]; ok {
cmd := []string{command}
if hasArgs {
cmd = append(cmd, parsedArgs...)
}
config.Cmd = cmd
} else if hasArgs {
d.logger.Println("[DEBUG] driver.docker: ignoring args because command not specified")
}
return docker.CreateContainerOptions{
Config: config,
HostConfig: hostConfig,
}, nil
}