func (s *SchedulerServer) Run(hks hyperkube.Interface, _ []string) error {
// get scheduler low-level config
sc := schedcfg.CreateDefaultConfig()
if s.SchedulerConfigFileName != "" {
f, err := os.Open(s.SchedulerConfigFileName)
if err != nil {
log.Fatalf("Cannot open scheduler config file: %v", err)
}
err = sc.Read(bufio.NewReader(f))
if err != nil {
log.Fatalf("Invalid scheduler config file: %v", err)
}
}
schedulerProcess, driverFactory, etcdClient, eid := s.bootstrap(hks, sc)
if s.EnableProfiling {
profile.InstallHandler(s.mux)
}
go runtime.Until(func() {
log.V(1).Info("Starting HTTP interface")
log.Error(http.ListenAndServe(net.JoinHostPort(s.Address.String(), strconv.Itoa(s.Port)), s.mux))
}, sc.HttpBindInterval.Duration, schedulerProcess.Terminal())
if s.HA {
validation := ha.ValidationFunc(validateLeadershipTransition)
srv := ha.NewCandidate(schedulerProcess, driverFactory, validation)
path := fmt.Sprintf(meta.DefaultElectionFormat, s.FrameworkName)
sid := uid.New(eid.Group(), "").String()
log.Infof("registering for election at %v with id %v", path, sid)
go election.Notify(election.NewEtcdMasterElector(etcdClient), path, sid, srv, nil)
} else {
log.Infoln("self-electing in non-HA mode")
schedulerProcess.Elect(driverFactory)
}
return s.awaitFailover(schedulerProcess, func() error { return s.failover(s.getDriver(), hks) })
}
func (s *SchedulerServer) prepareExecutorInfo(hks hyperkube.Interface) (*mesos.ExecutorInfo, *uid.UID, error) {
ci := &mesos.CommandInfo{
Shell: proto.Bool(false),
}
//TODO(jdef) these should be shared constants with km
const (
KM_EXECUTOR = "executor"
KM_PROXY = "proxy"
)
if s.ExecutorPath != "" {
uri, executorCmd := s.serveFrameworkArtifact(s.ExecutorPath)
ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)})
ci.Value = proto.String(fmt.Sprintf("./%s", executorCmd))
} else if !hks.FindServer(KM_EXECUTOR) {
return nil, nil, fmt.Errorf("either run this scheduler via km or else --executor-path is required")
} else {
if strings.Index(s.KMPath, "://") > 0 {
// URI could point directly to executable, e.g. hdfs:///km
// or else indirectly, e.g. http://acmestorage/tarball.tgz
// so we assume that for this case the command will always "km"
ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(s.KMPath), Executable: proto.Bool(true)})
ci.Value = proto.String("./km") // TODO(jdef) extract constant
} else if s.KMPath != "" {
uri, kmCmd := s.serveFrameworkArtifact(s.KMPath)
ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)})
ci.Value = proto.String(fmt.Sprintf("./%s", kmCmd))
} else {
uri, kmCmd := s.serveFrameworkArtifact(s.executable)
ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)})
ci.Value = proto.String(fmt.Sprintf("./%s", kmCmd))
}
ci.Arguments = append(ci.Arguments, KM_EXECUTOR)
}
if s.ProxyPath != "" {
uri, proxyCmd := s.serveFrameworkArtifact(s.ProxyPath)
ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)})
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--proxy-exec=./%s", proxyCmd))
} else if !hks.FindServer(KM_PROXY) {
return nil, nil, fmt.Errorf("either run this scheduler via km or else --proxy-path is required")
} else if s.ExecutorPath != "" {
return nil, nil, fmt.Errorf("proxy can only use km binary if executor does the same")
} // else, executor is smart enough to know when proxy-path is required, or to use km
//TODO(jdef): provide some way (env var?) for users to customize executor config
//TODO(jdef): set -address to 127.0.0.1 if `address` is 127.0.0.1
//TODO(jdef): propagate dockercfg from RootDirectory?
apiServerArgs := strings.Join(s.APIServerList, ",")
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--api-servers=%s", apiServerArgs))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--v=%d", s.ExecutorLogV))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--allow-privileged=%t", s.AllowPrivileged))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--suicide-timeout=%v", s.ExecutorSuicideTimeout))
if s.ExecutorBindall {
//TODO(jdef) determine whether hostname-override is really needed for bindall because
//it conflicts with kubelet node status checks/updates
//ci.Arguments = append(ci.Arguments, "--hostname-override=0.0.0.0")
ci.Arguments = append(ci.Arguments, "--address=0.0.0.0")
}
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--proxy-bindall=%v", s.ExecutorProxyBindall))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--run-proxy=%v", s.ExecutorRunProxy))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--cadvisor-port=%v", s.KubeletCadvisorPort))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--sync-frequency=%v", s.KubeletSyncFrequency))
if s.AuthPath != "" {
//TODO(jdef) should probably support non-local files, e.g. hdfs:///some/config/file
uri, basename := s.serveFrameworkArtifact(s.AuthPath)
ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri)})
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--auth-path=%s", basename))
}
appendOptional := func(name string, value string) {
if value != "" {
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--%s=%s", name, value))
}
}
if s.ClusterDNS != nil {
appendOptional("cluster-dns", s.ClusterDNS.String())
}
appendOptional("cluster-domain", s.ClusterDomain)
appendOptional("root-dir", s.KubeletRootDirectory)
appendOptional("docker-endpoint", s.KubeletDockerEndpoint)
appendOptional("pod-infra-container-image", s.KubeletPodInfraContainerImage)
appendOptional("host-network-sources", s.KubeletHostNetworkSources)
appendOptional("network-plugin", s.KubeletNetworkPluginName)
log.V(1).Infof("prepared executor command %q with args '%+v'", ci.GetValue(), ci.Arguments)
// Create mesos scheduler driver.
info := &mesos.ExecutorInfo{
Command: ci,
Name: proto.String(execcfg.DefaultInfoName),
Source: proto.String(execcfg.DefaultInfoSource),
}
// Check for staticPods
if s.StaticPodsConfigPath != "" {
bs, numberStaticPods, err := archive.ZipDir(s.StaticPodsConfigPath)
if err != nil {
return nil, nil, err
}
info.Data = bs
// Adjust the resource accounting for the executor.
// Currently each podTask accounts the default amount of resources.
// TODO(joerg84) adapt to actual resources specified by pods.
log.Infof("Detected %d staticPods in Configuration.", numberStaticPods)
info.Resources = []*mesos.Resource{
mutil.NewScalarResource("cpus", float64(numberStaticPods)*podtask.DefaultContainerCpus),
mutil.NewScalarResource("mem", float64(numberStaticPods)*podtask.DefaultContainerMem),
}
}
// calculate ExecutorInfo hash to be used for validating compatibility
// of ExecutorInfo's generated by other HA schedulers.
ehash := hashExecutorInfo(info)
eid := uid.New(ehash, execcfg.DefaultInfoID)
info.ExecutorId = &mesos.ExecutorID{Value: proto.String(eid.String())}
return info, eid, nil
}
func (s *SchedulerServer) prepareExecutorInfo(hks hyperkube.Interface) (*mesos.ExecutorInfo, *uid.UID, error) {
ci := &mesos.CommandInfo{
Shell: proto.Bool(false),
}
if s.ExecutorPath != "" {
uri, executorCmd := s.serveFrameworkArtifact(s.ExecutorPath)
ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)})
ci.Value = proto.String(fmt.Sprintf("./%s", executorCmd))
} else if !hks.FindServer(hyperkube.CommandMinion) {
return nil, nil, fmt.Errorf("either run this scheduler via km or else --executor-path is required")
} else {
if strings.Index(s.KMPath, "://") > 0 {
// URI could point directly to executable, e.g. hdfs:///km
// or else indirectly, e.g. http://acmestorage/tarball.tgz
// so we assume that for this case the command will always "km"
ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(s.KMPath), Executable: proto.Bool(true)})
ci.Value = proto.String("./km") // TODO(jdef) extract constant
} else if s.KMPath != "" {
uri, kmCmd := s.serveFrameworkArtifact(s.KMPath)
ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)})
ci.Value = proto.String(fmt.Sprintf("./%s", kmCmd))
} else {
uri, kmCmd := s.serveFrameworkArtifact(s.executable)
ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)})
ci.Value = proto.String(fmt.Sprintf("./%s", kmCmd))
}
ci.Arguments = append(ci.Arguments, hyperkube.CommandMinion)
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--run-proxy=%v", s.RunProxy))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--proxy-bindall=%v", s.ProxyBindall))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--proxy-logv=%d", s.ProxyLogV))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--max-log-size=%v", s.MinionLogMaxSize.String()))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--max-log-backups=%d", s.MinionLogMaxBackups))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--max-log-age=%d", s.MinionLogMaxAgeInDays))
}
//TODO(jdef): provide some way (env var?) for users to customize executor config
//TODO(jdef): set -address to 127.0.0.1 if `address` is 127.0.0.1
//TODO(jdef): propagate dockercfg from RootDirectory?
apiServerArgs := strings.Join(s.APIServerList, ",")
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--api-servers=%s", apiServerArgs))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--v=%d", s.ExecutorLogV)) // this also applies to the minion
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--allow-privileged=%t", s.AllowPrivileged))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--suicide-timeout=%v", s.ExecutorSuicideTimeout))
if s.ExecutorBindall {
//TODO(jdef) determine whether hostname-override is really needed for bindall because
//it conflicts with kubelet node status checks/updates
//ci.Arguments = append(ci.Arguments, "--hostname-override=0.0.0.0")
ci.Arguments = append(ci.Arguments, "--address=0.0.0.0")
}
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--cgroup-prefix=%v", s.ExecutorCgroupPrefix))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--cadvisor-port=%v", s.KubeletCadvisorPort))
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--sync-frequency=%v", s.KubeletSyncFrequency))
if s.AuthPath != "" {
//TODO(jdef) should probably support non-local files, e.g. hdfs:///some/config/file
uri, basename := s.serveFrameworkArtifact(s.AuthPath)
ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri)})
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--auth-path=%s", basename))
}
appendOptional := func(name string, value string) {
if value != "" {
ci.Arguments = append(ci.Arguments, fmt.Sprintf("--%s=%s", name, value))
}
}
if s.ClusterDNS != nil {
appendOptional("cluster-dns", s.ClusterDNS.String())
}
appendOptional("cluster-domain", s.ClusterDomain)
appendOptional("root-dir", s.KubeletRootDirectory)
appendOptional("docker-endpoint", s.KubeletDockerEndpoint)
appendOptional("pod-infra-container-image", s.KubeletPodInfraContainerImage)
appendOptional("host-network-sources", s.KubeletHostNetworkSources)
appendOptional("network-plugin", s.KubeletNetworkPluginName)
log.V(1).Infof("prepared executor command %q with args '%+v'", ci.GetValue(), ci.Arguments)
// Create mesos scheduler driver.
execInfo := &mesos.ExecutorInfo{
Command: ci,
Name: proto.String(execcfg.DefaultInfoName),
Source: proto.String(execcfg.DefaultInfoSource),
}
// Check for staticPods
var staticPodCPUs, staticPodMem float64
if s.StaticPodsConfigPath != "" {
bs, paths, err := archive.ZipDir(s.StaticPodsConfigPath)
if err != nil {
return nil, nil, err
}
// try to read pod files and sum resources
// TODO(sttts): don't terminate when static pods are broken, but skip them
// TODO(sttts): add a directory watch and tell running executors about updates
for _, podPath := range paths {
podJson, err := ioutil.ReadFile(podPath)
if err != nil {
return nil, nil, fmt.Errorf("error reading static pod spec: %v", err)
}
pod := api.Pod{}
err = json.Unmarshal(podJson, &pod)
if err != nil {
return nil, nil, fmt.Errorf("error parsing static pod spec at %v: %v", podPath, err)
}
// TODO(sttts): allow unlimited static pods as well and patch in the default resource limits
unlimitedCPU := mresource.LimitPodCPU(&pod, s.DefaultContainerCPULimit)
unlimitedMem := mresource.LimitPodMem(&pod, s.DefaultContainerMemLimit)
if unlimitedCPU {
return nil, nil, fmt.Errorf("found static pod without limit on cpu resources: %v", podPath)
}
if unlimitedMem {
return nil, nil, fmt.Errorf("found static pod without limit on memory resources: %v", podPath)
}
cpu := mresource.PodCPULimit(&pod)
mem := mresource.PodMemLimit(&pod)
log.V(2).Infof("reserving %.2f cpu shares and %.2f MB of memory to static pod %s", cpu, mem, pod.Name)
staticPodCPUs += float64(cpu)
staticPodMem += float64(mem)
}
// pass zipped pod spec to executor
execInfo.Data = bs
}
execInfo.Resources = []*mesos.Resource{
mutil.NewScalarResource("cpus", float64(executorCPUs)+staticPodCPUs),
mutil.NewScalarResource("mem", float64(executorMem)+staticPodMem),
}
// calculate ExecutorInfo hash to be used for validating compatibility
// of ExecutorInfo's generated by other HA schedulers.
ehash := hashExecutorInfo(execInfo)
eid := uid.New(ehash, execcfg.DefaultInfoID)
execInfo.ExecutorId = &mesos.ExecutorID{Value: proto.String(eid.String())}
return execInfo, eid, nil
}