func AskForPorts(portCount int) ResourceAsker { ret := func(resources []*mesos.Resource) ([]*mesos.Resource, *mesos.Resource, bool) { newResources := make([]*mesos.Resource, len(resources)) copy(newResources, resources) for idx, resource := range resources { if resource.GetName() == "ports" { ports := RangesToArray(resource.GetRanges().GetRange()) // Now we have to see if there are N ports if len(ports) >= portCount { var sliceLoc int // Calculate the slice where I'm taking ports: if len(ports)-portCount == 0 { sliceLoc = 0 } else { sliceLoc = rand.Intn(len(ports) - portCount) } takingPorts := make([]int64, portCount) copy(takingPorts, ports[sliceLoc:(sliceLoc+portCount)]) leavingPorts := make([]int64, len(ports)-portCount) copy(leavingPorts, ports[:sliceLoc]) copy(leavingPorts[sliceLoc:], ports[(sliceLoc+portCount):]) newResources[idx] = util.NewRangesResource("ports", ArrayToRanges(leavingPorts)) ask := util.NewRangesResource("ports", ArrayToRanges(takingPorts)) return newResources, ask, true } } } return resources, nil, false } return ret }
func (s *Scheduler) launchTask(driver scheduler.SchedulerDriver, offer *mesos.Offer) { taskName := fmt.Sprintf("syslog-%s", offer.GetSlaveId().GetValue()) taskId := &mesos.TaskID{ Value: proto.String(fmt.Sprintf("%s-%s", taskName, uuid())), } data, err := json.Marshal(Config) if err != nil { panic(err) //shouldn't happen } Logger.Debugf("Task data: %s", string(data)) tcpPort := uint64(s.getPort(Config.TcpPort, offer, -1)) udpPort := uint64(s.getPort(Config.UdpPort, offer, int(tcpPort))) task := &mesos.TaskInfo{ Name: proto.String(taskName), TaskId: taskId, SlaveId: offer.GetSlaveId(), Executor: s.createExecutor(offer, tcpPort, udpPort), Resources: []*mesos.Resource{ util.NewScalarResource("cpus", Config.Cpus), util.NewScalarResource("mem", Config.Mem), util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(tcpPort, tcpPort)}), util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(udpPort, udpPort)}), }, Data: data, Labels: utils.StringToLabels(s.labels), } s.cluster.Add(offer.GetSlaveId().GetValue(), task) driver.LaunchTasks([]*mesos.OfferID{offer.GetId()}, []*mesos.TaskInfo{task}, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) }
func (offerHelper *OfferHelper) apply(against *ResourceGroup, cpus float64, mem float64, disk float64, ports int, principal string, role string, persistenceID string, containerPath string) []*mesos.Resource { ask := []*mesos.Resource{} if cpus > 0 { against.Cpus = against.Cpus - cpus if principal != "" && role != "" { ask = append(ask, util.NewScalarResourceWithReservation("cpus", cpus, principal, role)) } else { ask = append(ask, util.NewScalarResource("cpus", cpus)) } } if mem > 0 { against.Mem = against.Mem - mem if principal != "" && role != "" { ask = append(ask, util.NewScalarResourceWithReservation("mem", mem, principal, role)) } else { ask = append(ask, util.NewScalarResource("mem", mem)) } } if disk > 0 { against.Disk = against.Disk - disk if principal != "" && role != "" && containerPath != "" && persistenceID != "" { ask = append(ask, util.NewVolumeResourceWithReservation(disk, containerPath, persistenceID, mesos.Volume_RW.Enum(), principal, role)) } else if principal != "" && role != "" { ask = append(ask, util.NewScalarResourceWithReservation("disk", disk, principal, role)) } else { ask = append(ask, util.NewScalarResource("disk", disk)) } } if ports > 0 { sliceLoc := 0 if len(against.Ports)-ports > 0 { sliceLoc = rand.Intn(len(against.Ports) - ports) } takingPorts := make([]int64, ports) copy(takingPorts, against.Ports[sliceLoc:(sliceLoc+ports)]) leavingPorts := make([]int64, len(against.Ports)-ports) copy(leavingPorts, against.Ports[:sliceLoc]) copy(leavingPorts[sliceLoc:], against.Ports[(sliceLoc+ports):]) against.Ports = leavingPorts if principal != "" && role != "" { ask = append(ask, util.AddResourceReservation(util.NewRangesResource("ports", ArrayToRanges(takingPorts)), principal, role)) } else { ask = append(ask, util.NewRangesResource("ports", ArrayToRanges(takingPorts))) } } return ask }
func TestGoodPortAsk(t *testing.T) { rand.Seed(10) assert := assert.New(t) offer := generateResourceOffer() askFun := AskForPorts(100) remaining, resourceAsk, success := askFun(offer) assert.Equal(true, success) assert.Equal(util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(31105, 31204)}), resourceAsk) remainingPorts := util.FilterResources(remaining, func(res *mesos.Resource) bool { return res.GetName() == "ports" }) assert.Equal([]*mesos.Resource{util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(31000, 31104), util.NewValueRange(31205, 32000)})}, remainingPorts) }
func createTaskInfo(task eremetic.Task, offer *mesosproto.Offer) (eremetic.Task, *mesosproto.TaskInfo) { task.FrameworkID = *offer.FrameworkId.Value task.SlaveID = *offer.SlaveId.Value task.Hostname = *offer.Hostname task.AgentIP = offer.GetUrl().GetAddress().GetIp() task.AgentPort = offer.GetUrl().GetAddress().GetPort() portMapping, portResources := buildPorts(task, offer) env := buildEnvironment(task, portMapping) taskInfo := &mesosproto.TaskInfo{ TaskId: &mesosproto.TaskID{Value: proto.String(task.ID)}, SlaveId: offer.SlaveId, Name: proto.String(task.Name), Command: buildCommandInfo(task, env), Container: &mesosproto.ContainerInfo{ Type: mesosproto.ContainerInfo_DOCKER.Enum(), Docker: &mesosproto.ContainerInfo_DockerInfo{ Image: proto.String(task.Image), ForcePullImage: proto.Bool(task.ForcePullImage), PortMappings: portMapping, Network: mesosproto.ContainerInfo_DockerInfo_BRIDGE.Enum(), }, Volumes: buildVolumes(task), }, Resources: []*mesosproto.Resource{ mesosutil.NewScalarResource("cpus", task.TaskCPUs), mesosutil.NewScalarResource("mem", task.TaskMem), mesosutil.NewRangesResource("ports", portResources), }, } return task, taskInfo }
func (s *MemcacheScheduler) newTaskPrototype(offer *mesos.Offer) *mesos.TaskInfo { taskID := s.tasksCreated s.tasksCreated++ portRange := getPortRange(offer) portRange.End = portRange.Begin return &mesos.TaskInfo{ TaskId: &mesos.TaskID{ Value: proto.String(fmt.Sprintf("Memcache-%d", taskID)), }, SlaveId: offer.SlaveId, Resources: []*mesos.Resource{ mesosutil.NewScalarResource("cpus", TASK_CPUS), mesosutil.NewScalarResource("mem", TASK_MEM), mesosutil.NewRangesResource("ports", []*mesos.Value_Range{portRange}), }, Container: &mesos.ContainerInfo{ Type: mesos.ContainerInfo_DOCKER.Enum(), Docker: &mesos.ContainerInfo_DockerInfo{ Image: &util.MEMCACHE_CONTAINER, Network: mesos.ContainerInfo_DockerInfo_BRIDGE.Enum(), PortMappings: []*mesos.ContainerInfo_DockerInfo_PortMapping{ &mesos.ContainerInfo_DockerInfo_PortMapping{ ContainerPort: &MEMCACHE_PORT, Protocol: &MEMCACHE_PROTOCOL, HostPort: &MEMCACHE_HOST_PORT, }, }, }, }, } }
func TestResource(t *testing.T) { mem := Resource(util.NewScalarResource("mem", 512)) if mem != "mem:512.00" { t.Errorf(`Resource(util.NewScalarResource("mem", 512)) != "mem:512.00"; actual %s`, mem) } ports := Resource(util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(31000, 32000)})) if ports != "ports:[31000..32000]" { t.Errorf(`Resource(util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(31000, 32000)})) != "ports:[31000..32000]"; actual %s`, ports) } ports = Resource(util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(4000, 7000), util.NewValueRange(31000, 32000)})) if ports != "ports:[4000..7000][31000..32000]" { t.Errorf(`Resource(util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(4000, 7000), util.NewValueRange(31000, 32000)})) != "ports:[4000..7000][31000..32000]"; actual %s`, ports) } }
func TestBadPortAsk(t *testing.T) { assert := assert.New(t) offer := []*mesos.Resource{util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(31000, 31000)})} _, _, success := AskForPorts(100)(offer) assert.Equal(false, success) }
func TestTotalPortAsk(t *testing.T) { assert := assert.New(t) askfun := AskForPorts(1) offer := []*mesos.Resource{util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(31000, 31000)})} newOffer, _, success := askfun(offer) newOffer[0].GetRanges().GetRange() assert.Equal(0, len(newOffer[0].GetRanges().GetRange())) assert.Equal(true, success) }
func generateResourceOffer() []*mesos.Resource { val := []*mesos.Resource{ util.NewScalarResource("cpus", 3), util.NewScalarResource("disk", 73590), util.NewScalarResource("mem", 1985), util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(31000, 32000)}), } return val }
func (sched *Scheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { for _, offer := range offers { taskId := &mesos.TaskID{ Value: proto.String(fmt.Sprintf("basicdocker-task-%d", time.Now().Unix())), } ports := util.FilterResources( offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "ports" }, ) if len(ports) > 0 && len(ports[0].GetRanges().GetRange()) > 0 { } else { return } task := &mesos.TaskInfo{ Name: proto.String(taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Container: &mesos.ContainerInfo{ Type: mesos.ContainerInfo_DOCKER.Enum(), Volumes: nil, Hostname: nil, Docker: &mesos.ContainerInfo_DockerInfo{ Image: &DOCKER_IMAGE_DEFAULT, Network: mesos.ContainerInfo_DockerInfo_BRIDGE.Enum(), }, }, Command: &mesos.CommandInfo{ Shell: proto.Bool(true), Value: proto.String("set -x ; /bin/date ; /bin/hostname ; sleep 200 ; echo done"), }, Executor: nil, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", getOfferCpu(offer)), util.NewScalarResource("mem", getOfferMem(offer)), util.NewRangesResource("ports", []*mesos.Value_Range{ util.NewValueRange( *ports[0].GetRanges().GetRange()[0].Begin, *ports[0].GetRanges().GetRange()[0].Begin+1, ), }), }, } log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) var tasks []*mesos.TaskInfo = []*mesos.TaskInfo{task} log.Infoln("Launching ", len(tasks), " tasks for offer", offer.Id.GetValue()) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) sched.tasksLaunched++ time.Sleep(time.Second) } }
// Offering some cpus and memory and the 8000-9000 port range func NewTestOffer(id string) *mesos.Offer { hostname := "some_hostname" cpus := util.NewScalarResource("cpus", 3.75) mem := util.NewScalarResource("mem", 940) var port8000 uint64 = 8000 var port9000 uint64 = 9000 ports8000to9000 := mesos.Value_Range{Begin: &port8000, End: &port9000} ports := util.NewRangesResource("ports", []*mesos.Value_Range{&ports8000to9000}) return &mesos.Offer{ Id: util.NewOfferID(id), Hostname: &hostname, SlaveId: util.NewSlaveID(hostname), Resources: []*mesos.Resource{cpus, mem, ports}, } }
func NewOffer(id string) *mesos.Offer { return &mesos.Offer{ Id: util.NewOfferID(id), FrameworkId: util.NewFrameworkID("test-etcd-framework"), SlaveId: util.NewSlaveID("slave-" + id), Hostname: proto.String("localhost"), Resources: []*mesos.Resource{ util.NewScalarResource("cpus", 1), util.NewScalarResource("mem", 256), util.NewScalarResource("disk", 4096), util.NewRangesResource("ports", []*mesos.Value_Range{ util.NewValueRange(uint64(0), uint64(65535)), }), }, } }
func (this *ElodinaTransportScheduler) launchNewTask(offers []*OfferAndResources) (*mesos.Offer, *mesos.TaskInfo) { for _, offer := range offers { configBlob, err := json.Marshal(this.config.ConsumerConfig) if err != nil { break } log.Logger.Debug("%v", offer) if this.hasEnoughResources(offer) { port := this.takePort(&offer.RemainingPorts) taskPort := &mesos.Value_Range{Begin: port, End: port} taskId := &mesos.TaskID{ Value: proto.String(fmt.Sprintf("elodina-mirror-%s-%d", *offer.Offer.Hostname, *port)), } cpuTaken := this.config.CpuPerTask * float64(this.config.ThreadsPerTask) memoryTaken := this.config.MemPerTask * float64(this.config.ThreadsPerTask) task := &mesos.TaskInfo{ Name: proto.String(taskId.GetValue()), TaskId: taskId, SlaveId: offer.Offer.SlaveId, Executor: this.createExecutor(len(this.taskIdToTaskState), *port), Resources: []*mesos.Resource{ util.NewScalarResource("cpus", cpuTaken), util.NewScalarResource("mem", memoryTaken), util.NewRangesResource("ports", []*mesos.Value_Range{taskPort}), }, Data: configBlob, } log.Logger.Debug("Prepared task: %s with offer %s for launch. Ports: %s", task.GetName(), offer.Offer.Id.GetValue(), taskPort) transport := NewElodinaTransport(fmt.Sprintf("http://%s:%d/assign", *offer.Offer.Hostname, *port), task, this.config.StaleDuration) this.taskIdToTaskState[taskId.GetValue()] = transport log.Logger.Debug("Prepared task: %s with offer %s for launch. Ports: %s", task.GetName(), offer.Offer.Id.GetValue(), taskPort) offer.RemainingPorts = offer.RemainingPorts[1:] offer.RemainingCpu -= cpuTaken offer.RemainingMemory -= memoryTaken return offer.Offer, task } else { log.Logger.Info("Not enough CPU and memory") } } return nil, nil }
func (t *task) build(slaveID string) { t.Command = &mesosproto.CommandInfo{Shell: proto.Bool(false)} t.Container = &mesosproto.ContainerInfo{ Type: mesosproto.ContainerInfo_DOCKER.Enum(), Docker: &mesosproto.ContainerInfo_DockerInfo{ Image: &t.config.Image, }, } switch t.config.HostConfig.NetworkMode { case "none": t.Container.Docker.Network = mesosproto.ContainerInfo_DockerInfo_NONE.Enum() case "host": t.Container.Docker.Network = mesosproto.ContainerInfo_DockerInfo_HOST.Enum() case "bridge", "": for containerPort, bindings := range t.config.HostConfig.PortBindings { for _, binding := range bindings { fmt.Println(containerPort) containerInfo := strings.SplitN(containerPort, "/", 2) fmt.Println(containerInfo[0], containerInfo[1]) containerPort, err := strconv.ParseUint(containerInfo[0], 10, 32) if err != nil { log.Warn(err) continue } hostPort, err := strconv.ParseUint(binding.HostPort, 10, 32) if err != nil { log.Warn(err) continue } protocol := "tcp" if len(containerInfo) == 2 { protocol = containerInfo[1] } t.Container.Docker.PortMappings = append(t.Container.Docker.PortMappings, &mesosproto.ContainerInfo_DockerInfo_PortMapping{ HostPort: proto.Uint32(uint32(hostPort)), ContainerPort: proto.Uint32(uint32(containerPort)), Protocol: proto.String(protocol), }) t.Resources = append(t.Resources, mesosutil.NewRangesResource("ports", []*mesosproto.Value_Range{mesosutil.NewValueRange(hostPort, hostPort)})) } } // TODO handle -P here t.Container.Docker.Network = mesosproto.ContainerInfo_DockerInfo_BRIDGE.Enum() default: log.Errorf("Unsupported network mode %q", t.config.HostConfig.NetworkMode) t.Container.Docker.Network = mesosproto.ContainerInfo_DockerInfo_BRIDGE.Enum() } if cpus := t.config.CpuShares; cpus > 0 { t.Resources = append(t.Resources, mesosutil.NewScalarResource("cpus", float64(cpus))) } if mem := t.config.Memory; mem > 0 { t.Resources = append(t.Resources, mesosutil.NewScalarResource("mem", float64(mem/1024/1024))) } if len(t.config.Cmd) > 0 && t.config.Cmd[0] != "" { t.Command.Value = &t.config.Cmd[0] } if len(t.config.Cmd) > 1 { t.Command.Arguments = t.config.Cmd[1:] } for key, value := range t.config.Labels { t.Container.Docker.Parameters = append(t.Container.Docker.Parameters, &mesosproto.Parameter{Key: proto.String("label"), Value: proto.String(fmt.Sprintf("%s=%s", key, value))}) } t.SlaveId = &mesosproto.SlaveID{Value: &slaveID} }
func TestWildcardHostPortMatching(t *testing.T) { t.Parallel() task := fakePodTask("foo", nil, nil) pod := &task.Pod offer := &mesos.Offer{} mapping, err := WildcardMapper(task, offer) if err != nil { t.Fatal(err) } if len(mapping) > 0 { t.Fatalf("Found mappings for an empty offer and a pod without ports: %v", pod) } //-- offer = &mesos.Offer{ Resources: []*mesos.Resource{ newPortsResource("*", 1, 1), }, } mapping, err = WildcardMapper(task, offer) if err != nil { t.Fatal(err) } if len(mapping) > 0 { t.Fatalf("Found mappings for a pod without ports: %v", pod) } //-- pod.Spec = api.PodSpec{ Containers: []api.Container{{ Ports: []api.ContainerPort{{ HostPort: 123, }}, }}, } task, err = New(api.NewDefaultContext(), "", pod, &mesos.ExecutorInfo{}, nil, nil) if err != nil { t.Fatal(err) } mapping, err = WildcardMapper(task, offer) if err == nil { t.Fatalf("expected error instead of mappings: %#v", mapping) } else if err, _ := err.(*PortAllocationError); err == nil { t.Fatal("Expected port allocation error") } else if !(len(err.Ports) == 1 && err.Ports[0] == 123) { t.Fatal("Expected port allocation error for host port 123") } //-- pod.Spec = api.PodSpec{ Containers: []api.Container{{ Ports: []api.ContainerPort{{ HostPort: 0, }, { HostPort: 123, }}, }}, } task, err = New(api.NewDefaultContext(), "", pod, &mesos.ExecutorInfo{}, nil, nil) if err != nil { t.Fatal(err) } mapping, err = WildcardMapper(task, offer) if err, _ := err.(*PortAllocationError); err == nil { t.Fatal("Expected port allocation error") } else if !(len(err.Ports) == 1 && err.Ports[0] == 123) { t.Fatal("Expected port allocation error for host port 123") } //-- pod.Spec = api.PodSpec{ Containers: []api.Container{{ Ports: []api.ContainerPort{{ HostPort: 0, }, { HostPort: 1, }}, }}, } task, err = New(api.NewDefaultContext(), "", pod, &mesos.ExecutorInfo{}, nil, nil) if err != nil { t.Fatal(err) } mapping, err = WildcardMapper(task, offer) if err, _ := err.(*PortAllocationError); err == nil { t.Fatal("Expected port allocation error") } else if len(err.Ports) != 0 { t.Fatal("Expected port allocation error for wildcard port") } //-- offer = &mesos.Offer{ Resources: []*mesos.Resource{ newPortsResource("*", 1, 2), }, } mapping, err = WildcardMapper(task, offer) if err != nil { t.Fatal(err) } else if len(mapping) != 2 { t.Fatal("Expected both ports allocated") } valid := 0 for _, entry := range mapping { if entry.ContainerIdx == 0 && entry.PortIdx == 0 && entry.OfferPort == 2 { valid++ } if entry.ContainerIdx == 0 && entry.PortIdx == 1 && entry.OfferPort == 1 { valid++ } } if valid < 2 { t.Fatalf("Expected 2 valid port mappings, not %d", valid) } //-- port mapping in case of multiple discontinuous port ranges in mesos offer pod.Spec = api.PodSpec{ Containers: []api.Container{{ Ports: []api.ContainerPort{{ HostPort: 0, }, { HostPort: 0, }}, }}, } task, err = New(api.NewDefaultContext(), "", pod, &mesos.ExecutorInfo{}, nil, nil) if err != nil { t.Fatal(err) } offer = &mesos.Offer{ Resources: []*mesos.Resource{ mesosutil.NewRangesResource("ports", []*mesos.Value_Range{mesosutil.NewValueRange(1, 1), mesosutil.NewValueRange(3, 5)}), }, } mapping, err = WildcardMapper(task, offer) if err != nil { t.Fatal(err) } else if len(mapping) != 2 { t.Fatal("Expected both ports allocated") } valid = 0 for _, entry := range mapping { if entry.ContainerIdx == 0 && entry.PortIdx == 0 && entry.OfferPort == 1 { valid++ } if entry.ContainerIdx == 0 && entry.PortIdx == 1 && entry.OfferPort == 3 { valid++ } } if valid < 2 { t.Fatalf("Expected 2 valid port mappings, not %d", valid) } }
// mesos.Scheduler interface method. // Invoked when resources have been offered to this framework. func (this *TransformScheduler) ResourceOffers(driver scheduler.SchedulerDriver, offers []*mesos.Offer) { fmt.Println("Received offers") if int(this.runningInstances) > this.config.Instances { toKill := int(this.runningInstances) - this.config.Instances for i := 0; i < toKill; i++ { driver.KillTask(this.tasks[i]) } this.tasks = this.tasks[toKill:] } offersAndTasks := make(map[*mesos.Offer][]*mesos.TaskInfo) for _, offer := range offers { cpus := getScalarResources(offer, "cpus") mems := getScalarResources(offer, "mem") ports := getRangeResources(offer, "ports") remainingCpus := cpus remainingMems := mems var tasks []*mesos.TaskInfo for int(this.getRunningInstances()) < this.config.Instances && this.config.CpuPerTask <= remainingCpus && this.config.MemPerTask <= remainingMems && len(ports) > 0 { port := this.takePort(&ports) taskPort := &mesos.Value_Range{Begin: port, End: port} taskId := &mesos.TaskID{ Value: proto.String(fmt.Sprintf("golang-%s-%d", *offer.Hostname, *port)), } task := &mesos.TaskInfo{ Name: proto.String(taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Executor: this.createExecutor(this.getRunningInstances(), *port), Resources: []*mesos.Resource{ util.NewScalarResource("cpus", float64(this.config.CpuPerTask)), util.NewScalarResource("mem", float64(this.config.MemPerTask)), util.NewRangesResource("ports", []*mesos.Value_Range{taskPort}), }, } fmt.Printf("Prepared task: %s with offer %s for launch. Ports: %s\n", task.GetName(), offer.Id.GetValue(), taskPort) tasks = append(tasks, task) remainingCpus -= this.config.CpuPerTask remainingMems -= this.config.MemPerTask ports = ports[1:] this.tasks = append(this.tasks, taskId) this.incRunningInstances() } fmt.Printf("Launching %d tasks for offer %s\n", len(tasks), offer.Id.GetValue()) offersAndTasks[offer] = tasks } unlaunchedTasks := this.config.Instances - int(this.getRunningInstances()) if unlaunchedTasks > 0 { fmt.Printf("There are still %d tasks to be launched and no more resources are available.", unlaunchedTasks) } for _, offer := range offers { tasks := offersAndTasks[offer] driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)}) } }
// Build method builds the task func (t *Task) Build(slaveID string, offers map[string]*mesosproto.Offer) { t.Command = &mesosproto.CommandInfo{Shell: proto.Bool(false)} t.Container = &mesosproto.ContainerInfo{ Type: mesosproto.ContainerInfo_DOCKER.Enum(), Docker: &mesosproto.ContainerInfo_DockerInfo{ Image: &t.config.Image, }, } if t.config.Hostname != "" { t.Container.Hostname = proto.String(t.config.Hostname) if t.config.Domainname != "" { t.Container.Hostname = proto.String(t.config.Hostname + "." + t.config.Domainname) } } switch t.config.HostConfig.NetworkMode { case "none": t.Container.Docker.Network = mesosproto.ContainerInfo_DockerInfo_NONE.Enum() case "host": t.Container.Docker.Network = mesosproto.ContainerInfo_DockerInfo_HOST.Enum() case "default", "bridge", "": var ports []uint64 for _, offer := range offers { ports = append(ports, getPorts(offer)...) } for containerProtoPort, bindings := range t.config.HostConfig.PortBindings { for _, binding := range bindings { containerInfo := strings.SplitN(containerProtoPort, "/", 2) containerPort, err := strconv.ParseUint(containerInfo[0], 10, 32) if err != nil { log.Warn(err) continue } var hostPort uint64 if binding.HostPort != "" { hostPort, err = strconv.ParseUint(binding.HostPort, 10, 32) if err != nil { log.Warn(err) continue } } else if len(ports) > 0 { hostPort = ports[0] ports = ports[1:] } if hostPort == 0 { log.Warn("cannot find port to bind on the host") continue } protocol := "tcp" if len(containerInfo) == 2 { protocol = containerInfo[1] } t.Container.Docker.PortMappings = append(t.Container.Docker.PortMappings, &mesosproto.ContainerInfo_DockerInfo_PortMapping{ HostPort: proto.Uint32(uint32(hostPort)), ContainerPort: proto.Uint32(uint32(containerPort)), Protocol: proto.String(protocol), }) t.Resources = append(t.Resources, mesosutil.NewRangesResource("ports", []*mesosproto.Value_Range{mesosutil.NewValueRange(hostPort, hostPort)})) } } // TODO handle -P here t.Container.Docker.Network = mesosproto.ContainerInfo_DockerInfo_BRIDGE.Enum() default: log.Errorf("Unsupported network mode %q", t.config.HostConfig.NetworkMode) t.Container.Docker.Network = mesosproto.ContainerInfo_DockerInfo_BRIDGE.Enum() } if cpus := t.config.CpuShares; cpus > 0 { t.Resources = append(t.Resources, mesosutil.NewScalarResource("cpus", float64(cpus))) } if mem := t.config.Memory; mem > 0 { t.Resources = append(t.Resources, mesosutil.NewScalarResource("mem", float64(mem/1024/1024))) } if len(t.config.Cmd) > 0 && t.config.Cmd[0] != "" { t.Command.Value = &t.config.Cmd[0] } if len(t.config.Cmd) > 1 { t.Command.Arguments = t.config.Cmd[1:] } for key, value := range t.config.Labels { t.Container.Docker.Parameters = append(t.Container.Docker.Parameters, &mesosproto.Parameter{Key: proto.String("label"), Value: proto.String(fmt.Sprintf("%s=%s", key, value))}) } for _, value := range t.config.Env { t.Container.Docker.Parameters = append(t.Container.Docker.Parameters, &mesosproto.Parameter{Key: proto.String("env"), Value: proto.String(value)}) } if !t.config.AttachStdin && !t.config.AttachStdout && !t.config.AttachStderr { t.Container.Docker.Parameters = append(t.Container.Docker.Parameters, &mesosproto.Parameter{Key: proto.String("label"), Value: proto.String(fmt.Sprintf("%s=true", cluster.SwarmLabelNamespace+".mesos.detach"))}) } t.SlaveId = &mesosproto.SlaveID{Value: &slaveID} }
func TestWildcardHostPortMatching(t *testing.T) { pod := &api.Pod{ ObjectMeta: api.ObjectMeta{ Name: "foo", Namespace: "default", }, } offer := &mesos.Offer{} mapping, err := WildcardMapper(pod, []string{"*"}, offer) if err != nil { t.Fatal(err) } if len(mapping) > 0 { t.Fatalf("Found mappings for an empty offer and a pod without ports: %v", pod) } //-- offer = &mesos.Offer{ Resources: []*mesos.Resource{ resources.NewPorts("*", 1, 1), }, } mapping, err = WildcardMapper(pod, []string{"*"}, offer) if err != nil { t.Fatal(err) } if len(mapping) > 0 { t.Fatalf("Found mappings for a pod without ports: %v", pod) } //-- pod.Spec = api.PodSpec{ Containers: []api.Container{{ Ports: []api.ContainerPort{{ HostPort: 123, }}, }}, } mapping, err = WildcardMapper(pod, []string{"*"}, offer) if err == nil { t.Fatalf("expected error instead of mappings: %#v", mapping) } else if err, _ := err.(*PortAllocationError); err == nil { t.Fatal("Expected port allocation error") } else if !(len(err.Ports) == 1 && err.Ports[0] == 123) { t.Fatal("Expected port allocation error for host port 123") } //-- pod.Spec = api.PodSpec{ Containers: []api.Container{{ Ports: []api.ContainerPort{{ HostPort: 0, }, { HostPort: 123, }}, }}, } mapping, err = WildcardMapper(pod, []string{"*"}, offer) if err, _ := err.(*PortAllocationError); err == nil { t.Fatal("Expected port allocation error") } else if !(len(err.Ports) == 1 && err.Ports[0] == 123) { t.Fatal("Expected port allocation error for host port 123") } //-- pod.Spec = api.PodSpec{ Containers: []api.Container{{ Ports: []api.ContainerPort{{ HostPort: 0, }, { HostPort: 1, }}, }}, } mapping, err = WildcardMapper(pod, []string{"*"}, offer) if err, _ := err.(*PortAllocationError); err == nil { t.Fatal("Expected port allocation error") } else if len(err.Ports) != 0 { t.Fatal("Expected port allocation error for wildcard port") } //-- offer = &mesos.Offer{ Resources: []*mesos.Resource{ resources.NewPorts("*", 1, 2), }, } mapping, err = WildcardMapper(pod, []string{"*"}, offer) if err != nil { t.Fatal(err) } else if len(mapping) != 2 { t.Fatal("Expected both ports allocated") } valid := 0 for _, entry := range mapping { if entry.ContainerIdx == 0 && entry.PortIdx == 0 && entry.OfferPort == 2 { valid++ } if entry.ContainerIdx == 0 && entry.PortIdx == 1 && entry.OfferPort == 1 { valid++ } } if valid < 2 { t.Fatalf("Expected 2 valid port mappings, not %d", valid) } //-- port mapping in case of multiple discontinuous port ranges in mesos offer pod.Spec = api.PodSpec{ Containers: []api.Container{{ Ports: []api.ContainerPort{{ HostPort: 0, }, { HostPort: 0, }}, }}, } offer = &mesos.Offer{ Resources: []*mesos.Resource{ mesosutil.NewRangesResource("ports", []*mesos.Value_Range{mesosutil.NewValueRange(1, 1), mesosutil.NewValueRange(3, 5)}), }, } mapping, err = WildcardMapper(pod, []string{"*"}, offer) if err != nil { t.Fatal(err) } else if len(mapping) != 2 { t.Fatal("Expected both ports allocated") } valid = 0 for _, entry := range mapping { if entry.ContainerIdx == 0 && entry.PortIdx == 0 && entry.OfferPort == 1 { valid++ } if entry.ContainerIdx == 0 && entry.PortIdx == 1 && entry.OfferPort == 3 { valid++ } } if valid < 2 { t.Fatalf("Expected 2 valid port mappings, not %d", valid) } }
func TestResources(t *testing.T) { resources := Resources([]*mesos.Resource{util.NewScalarResource("cpus", 4), util.NewScalarResource("mem", 512), util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(31000, 32000)})}) if !strings.Contains(resources, "cpus") { t.Errorf(`%s does not contain "cpus"`, resources) } if !strings.Contains(resources, "mem") { t.Errorf(`%s does not contain "mem"`, resources) } if !strings.Contains(resources, "ports") { t.Errorf(`%s does not contain "ports"`, resources) } }
func TestOffers(t *testing.T) { offer1 := util.NewOffer(util.NewOfferID("487c73d8-9951-f23c-34bd-8085bfd30c49"), util.NewFrameworkID("20150903-065451-84125888-5050-10715-0053"), util.NewSlaveID("20150903-065451-84125888-5050-10715-S1"), "slave0") offer1.Resources = []*mesos.Resource{util.NewScalarResource("cpus", 4), util.NewScalarResource("mem", 512), util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(31000, 32000)})} offer2 := util.NewOffer(util.NewOfferID("26d5b34c-ef81-638d-5ad5-32c743c9c033"), util.NewFrameworkID("20150903-065451-84125888-5050-10715-0037"), util.NewSlaveID("20150903-065451-84125888-5050-10715-S0"), "master") offer2.Resources = []*mesos.Resource{util.NewScalarResource("cpus", 2), util.NewScalarResource("mem", 1024), util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(4000, 7000)})} offer2.Attributes = []*mesos.Attribute{&mesos.Attribute{ Name: proto.String("rack"), Type: mesos.Value_SCALAR.Enum(), Scalar: &mesos.Value_Scalar{Value: proto.Float64(2)}, }} offers := Offers([]*mesos.Offer{offer1, offer2}) if len(strings.Split(offers, "\n")) != 2 { t.Errorf("Offers([]*mesos.Offer{offer1, offer2}) should contain two offers split by new line, actual: %s", offers) } }
func TestOffer(t *testing.T) { offer := util.NewOffer(util.NewOfferID("487c73d8-9951-f23c-34bd-8085bfd30c49"), util.NewFrameworkID("20150903-065451-84125888-5050-10715-0053"), util.NewSlaveID("20150903-065451-84125888-5050-10715-S1"), "slave0") if Offer(offer) != "slave0#30c49" { t.Errorf(`util.NewOffer(util.NewOfferID("487c73d8-9951-f23c-34bd-8085bfd30c49"), util.NewFrameworkID("20150903-065451-84125888-5050-10715-0053"), util.NewSlaveID("20150903-065451-84125888-5050-10715-S1"), "slave0") != "slave0#30c49"; actual %s`, Offer(offer)) } offer.Resources = []*mesos.Resource{util.NewScalarResource("cpus", 4), util.NewScalarResource("mem", 512), util.NewRangesResource("ports", []*mesos.Value_Range{util.NewValueRange(31000, 32000)})} if Offer(offer) != "slave0#30c49 cpus:4.00 mem:512.00 ports:[31000..32000]" { t.Errorf("Expected slave0#30c49 cpus:4.00 mem:512.00 ports:[31000..32000]; actual %s", Offer(offer)) } offer.Attributes = []*mesos.Attribute{&mesos.Attribute{ Name: proto.String("rack"), Type: mesos.Value_SCALAR.Enum(), Scalar: &mesos.Value_Scalar{Value: proto.Float64(2)}, }} if Offer(offer) != "slave0#30c49 cpus:4.00 mem:512.00 ports:[31000..32000] rack:2.00" { t.Errorf("Expected slave0#30c49 cpus:4.00 mem:512.00 ports:[31000..32000] rack:2.00; actual %s", Offer(offer)) } offer.Resources = nil if Offer(offer) != "slave0#30c49 rack:2.00" { t.Errorf("Expected slave0#30c49 rack:2.00; actual %s", Offer(offer)) } }
func TestGrowToDesiredAfterReconciliation(t *gotesting.T) { testScheduler := NewEtcdScheduler(3, 0, 0, true, []*mesos.CommandInfo_URI{}, false, 4096, 1, 256) reconciliation := map[string]string{ "etcd-1": "slave-1", "etcd-2": "slave-2", } testScheduler.reconciliationInfoFunc = func([]string, string, string) (map[string]string, error) { return reconciliation, nil } testScheduler.updateReconciliationInfoFunc = func(info map[string]string, _ []string, _ string, _ string) error { reconciliation = info return nil } testScheduler.masterInfo = util.NewMasterInfo("master-1", 0, 0) mockdriver := &MockSchedulerDriver{ runningStatuses: make(chan *mesos.TaskStatus, 10), scheduler: testScheduler, } testScheduler.state = Mutable testScheduler.healthCheck = func(map[string]*config.Node) error { return nil } // Push more than enough offers to shoot self in foot if unchecked. for _, offer := range []*mesos.Offer{ NewOffer("1"), NewOffer("2"), NewOffer("3"), } { testScheduler.offerCache.Push(offer) } memberList := config.ClusterMemberList{ Members: []httptypes.Member{ { ID: "1", Name: "etcd-1", PeerURLs: nil, ClientURLs: nil, }, { ID: "2", Name: "etcd-2", PeerURLs: nil, ClientURLs: nil, }, }, } _, port1, err := emtesting.NewTestEtcdServer(t, memberList) if err != nil { t.Fatalf("Failed to create test etcd server: %s", err) } _, port2, err := emtesting.NewTestEtcdServer(t, memberList) if err != nil { t.Fatalf("Failed to create test etcd server: %s", err) } // Valid reconciled tasks should be added to the running list. mockdriver.On( "ReconcileTasks", 0, ).Return(mesos.Status_DRIVER_RUNNING, nil).Once() for _, taskStatus := range []*mesos.TaskStatus{ util.NewTaskStatus( util.NewTaskID("etcd-1 localhost 0 "+strconv.Itoa(int(port1))+" 0"), mesos.TaskState_TASK_RUNNING, ), util.NewTaskStatus( util.NewTaskID("etcd-2 localhost 0 "+strconv.Itoa(int(port2))+" 0"), mesos.TaskState_TASK_RUNNING, ), } { mockdriver.runningStatuses <- taskStatus } // Scheduler should grow cluster to desired number of nodes. offer := NewOffer("1") mockdriver.On( "LaunchTasks", []*mesos.OfferID{ offer.Id, }, []*mesos.TaskInfo{ { Resources: []*mesos.Resource{ util.NewScalarResource("cpus", 1), util.NewScalarResource("mem", 256), util.NewScalarResource("disk", 4096), util.NewRangesResource("ports", []*mesos.Value_Range{ util.NewValueRange(uint64(0), uint64(2)), }), }, }, }, &mesos.Filters{ RefuseSeconds: proto.Float64(1), }, ).Return(mesos.Status_DRIVER_RUNNING, nil).Once() // Simulate failover, registration and time passing. mockdriver.ReconcileTasks([]*mesos.TaskStatus{}) testScheduler.launchOne(mockdriver) testScheduler.launchOne(mockdriver) testScheduler.launchOne(mockdriver) testScheduler.launchOne(mockdriver) testScheduler.launchOne(mockdriver) assert.Equal(t, 3, len(testScheduler.running), "Scheduler should reconcile tasks properly.") mockdriver.AssertExpectations(t) }
func (s *visghsScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { if (s.nodesLaunched - s.nodesErrored) >= s.nodeTasks { log.Info("decline all of the offers since all of our tasks are already launched") ids := make([]*mesos.OfferID, len(offers)) for i, offer := range offers { ids[i] = offer.Id } driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(120)}) return } for _, offer := range offers { cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "cpus" }) cpus := 0.0 for _, res := range cpuResources { cpus += res.GetScalar().GetValue() } memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "mem" }) mems := 0.0 for _, res := range memResources { mems += res.GetScalar().GetValue() } portResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool { return res.GetName() == "ports" }) ports := []*mesos.Value_Range{} portsCount := uint64(0) for _, res := range portResources { for _, rs := range res.GetRanges().GetRange() { ports = append(ports, rs) portsCount += 1 + rs.GetEnd() - rs.GetBegin() } } log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems, " ports=", ports) remainingCpus := cpus remainingMems := mems remainingPorts := ports remainingPortsCount := portsCount // account for executor resources if there's not an executor already running on the slave if len(offer.ExecutorIds) == 0 { remainingCpus -= CPUS_PER_EXECUTOR remainingMems -= MEM_PER_EXECUTOR } var tasks []*mesos.TaskInfo for (s.nodesLaunched-s.nodesErrored) < s.nodeTasks && CPUS_PER_TASK <= remainingCpus && MEM_PER_TASK <= remainingMems && PORTS_PER_TASK <= remainingPortsCount { log.Infoln("Ports <", remainingPortsCount, remainingPorts) s.nodesLaunched++ taskId := &mesos.TaskID{ Value: proto.String(strconv.Itoa(s.nodesLaunched)), } taskPorts := []*mesos.Value_Range{} leftOverPorts := []*mesos.Value_Range{} for t := 0; t < PORTS_PER_TASK; t++ { if len(remainingPorts) < 1 { // failed to allocate port, oh no! } ps := remainingPorts[0] //take the first port from the first pb := ps.GetBegin() pe := ps.GetEnd() //Create one range per port we need, it's easier this way tp := mesos.Value_Range{} p := pb tp.Begin = &p tp.End = &p taskPorts = append(taskPorts, &tp) pb++ if pb <= pe { rpb := pb rpe := pe rtp := mesos.Value_Range{Begin: &rpb, End: &rpe} leftOverPorts = append(leftOverPorts, &rtp) } for _, ps := range remainingPorts[1:] { leftOverPorts = append(leftOverPorts, ps) } } radiaPort := (uint32)(taskPorts[0].GetBegin()) task := &mesos.TaskInfo{ Name: proto.String("visghs-node-" + taskId.GetValue()), TaskId: taskId, SlaveId: offer.SlaveId, Executor: s.nexec, Discovery: &mesos.DiscoveryInfo{ Name: proto.String("visghs"), //Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(), Visibility: mesos.DiscoveryInfo_FRAMEWORK.Enum(), Ports: &mesos.Ports{ Ports: []*mesos.Port{ {Protocol: proto.String("UDP"), Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(), Name: proto.String("udpprobe"), Number: proto.Uint32(radiaPort)}, {Protocol: proto.String("TCP"), Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(), Name: proto.String("radiarpc"), Number: proto.Uint32(radiaPort)}, }, }, }, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", CPUS_PER_TASK), util.NewScalarResource("mem", MEM_PER_TASK), util.NewRangesResource("ports", taskPorts), }, } log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue()) tasks = append(tasks, task) remainingCpus -= CPUS_PER_TASK remainingMems -= MEM_PER_TASK remainingPorts = leftOverPorts remainingPortsCount -= PORTS_PER_TASK } log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue()) driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)}) } }
// TODO(tyler) split this long function up! func (s *EtcdScheduler) launchOne(driver scheduler.SchedulerDriver) { // Always ensure we've pruned any dead / unmanaged nodes before // launching new ones, or we may overconfigure the ensemble such // that it can not make progress if the next launch fails. err := s.Prune() if err != nil { log.Errorf("Failed to remove stale cluster members: %s", err) return } if !s.shouldLaunch(driver) { log.Infoln("Skipping launch attempt for now.") return } // validOffer filters out offers that are no longer // desirable, even though they may have been when // they were enqueued. validOffer := func(offer *mesos.Offer) bool { runningCopy := s.RunningCopy() for _, etcdConfig := range runningCopy { if etcdConfig.SlaveID == offer.SlaveId.GetValue() { if s.singleInstancePerSlave { log.Info("Skipping offer: already running on this slave.") return false } } } return true } // Issue BlockingPop until we get back an offer we can use. var offer *mesos.Offer for { offer = s.offerCache.BlockingPop() if validOffer(offer) { break } else { s.decline(driver, offer) } } // Do this again because BlockingPop may have taken a long time. if !s.shouldLaunch(driver) { log.Infoln("Skipping launch attempt for now.") s.decline(driver, offer) return } // TODO(tyler) this is a broken hack resources := parseOffer(offer) lowest := *resources.ports[0].Begin rpcPort := lowest clientPort := lowest + 1 httpPort := lowest + 2 s.mut.Lock() var clusterType string if len(s.running) == 0 { clusterType = "new" } else { clusterType = "existing" } s.highestInstanceID++ name := "etcd-" + strconv.FormatInt(s.highestInstanceID, 10) node := &config.Node{ Name: name, Host: *offer.Hostname, RPCPort: rpcPort, ClientPort: clientPort, ReseedPort: httpPort, Type: clusterType, SlaveID: offer.GetSlaveId().GetValue(), } running := []*config.Node{node} for _, r := range s.running { running = append(running, r) } serializedNodes, err := json.Marshal(running) log.Infof("Serialized running: %+v", string(serializedNodes)) if err != nil { log.Errorf("Could not serialize running list: %v", err) // This Unlock is not deferred because the test implementation of LaunchTasks // calls this scheduler's StatusUpdate method, causing the test to deadlock. s.decline(driver, offer) s.mut.Unlock() return } configSummary := node.String() taskID := &mesos.TaskID{Value: &configSummary} executor := s.newExecutorInfo(node, s.executorUris) task := &mesos.TaskInfo{ Data: serializedNodes, Name: proto.String("etcd-server"), TaskId: taskID, SlaveId: offer.SlaveId, Executor: executor, Resources: []*mesos.Resource{ util.NewScalarResource("cpus", s.cpusPerTask), util.NewScalarResource("mem", s.memPerTask), util.NewScalarResource("disk", s.diskPerTask), util.NewRangesResource("ports", []*mesos.Value_Range{ util.NewValueRange(uint64(rpcPort), uint64(httpPort)), }), }, Discovery: &mesos.DiscoveryInfo{ Visibility: mesos.DiscoveryInfo_EXTERNAL.Enum(), Name: proto.String("etcd-server"), Ports: &mesos.Ports{ Ports: []*mesos.Port{ &mesos.Port{ Number: proto.Uint32(uint32(rpcPort)), Protocol: proto.String("tcp"), }, // HACK: "client" is not a real SRV protocol. This is so // that we can have etcd proxies use srv discovery on the // above tcp name. Mesos-dns does not yet care about // names for DiscoveryInfo. When it does, we should // create a name for clients to use. We want to keep // the rpcPort accessible at _etcd-server._tcp.<fwname>.mesos &mesos.Port{ Number: proto.Uint32(uint32(clientPort)), Protocol: proto.String("client"), }, }, }, }, } log.Infof( "Prepared task: %s with offer %s for launch", task.GetName(), offer.Id.GetValue(), ) log.Info("Launching etcd node.") tasks := []*mesos.TaskInfo{task} s.pending[node.Name] = struct{}{} // This Unlock is not deferred because the test implementation of LaunchTasks // calls this scheduler's StatusUpdate method, causing the test to deadlock. s.mut.Unlock() atomic.AddUint32(&s.Stats.LaunchedServers, 1) driver.LaunchTasks( []*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{ RefuseSeconds: proto.Float64(1), }, ) }