func Test_StaticPods(t *testing.T) { assert := assert.New(t) // create static pods config files, spod1 on toplevel and spod2 in a directory "dir" staticPodsConfigPath, err := ioutil.TempDir(os.TempDir(), "executor-k8sm-archive") assert.NoError(err) defer os.RemoveAll(staticPodsConfigPath) spod1, err := os.Create(filepath.Join(staticPodsConfigPath, "spod1.json")) assert.NoError(err) _, err = spod1.WriteString("content1") assert.NoError(err) err = os.Mkdir(filepath.Join(staticPodsConfigPath, "dir"), 0755) assert.NoError(err) spod2, err := os.Create(filepath.Join(staticPodsConfigPath, "dir", "spod2.json")) assert.NoError(err) _, err = spod2.WriteString("content2") assert.NoError(err) // archive config files data, fileNum, err := archive.ZipDir(staticPodsConfigPath) assert.NoError(err) assert.Equal(2, fileNum) // unarchive config files zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) assert.NoError(err) fileNames := []string{} for _, f := range zr.File { if !f.FileInfo().IsDir() { fileNames = append(fileNames, f.Name) } } assert.Contains(fileNames, "spod1.json") assert.Contains(fileNames, "dir/spod2.json") }
func (s *SchedulerServer) prepareExecutorInfo(hks hyperqing.Interface) (*mesos.ExecutorInfo, *uid.UID, error) { ci := &mesos.CommandInfo{ Shell: proto.Bool(false), } //TODO(jdef) these should be shared constants with km const ( KM_EXECUTOR = "executor" KM_PROXY = "proxy" ) if s.ExecutorPath != "" { uri, executorCmd := s.serveFrameworkArtifact(s.ExecutorPath) ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)}) ci.Value = proto.String(fmt.Sprintf("./%s", executorCmd)) } else if !hks.FindServer(KM_EXECUTOR) { return nil, nil, fmt.Errorf("either run this scheduler via km or else --executor-path is required") } else { if strings.Index(s.KMPath, "://") > 0 { // URI could point directly to executable, e.g. hdfs:///km // or else indirectly, e.g. http://acmestorage/tarball.tgz // so we assume that for this case the command will always "km" ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(s.KMPath), Executable: proto.Bool(true)}) ci.Value = proto.String("./km") // TODO(jdef) extract constant } else if s.KMPath != "" { uri, kmCmd := s.serveFrameworkArtifact(s.KMPath) ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)}) ci.Value = proto.String(fmt.Sprintf("./%s", kmCmd)) } else { uri, kmCmd := s.serveFrameworkArtifact(s.executable) ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)}) ci.Value = proto.String(fmt.Sprintf("./%s", kmCmd)) } ci.Arguments = append(ci.Arguments, KM_EXECUTOR) } if s.ProxyPath != "" { uri, proxyCmd := s.serveFrameworkArtifact(s.ProxyPath) ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(true)}) ci.Arguments = append(ci.Arguments, fmt.Sprintf("--proxy-exec=./%s", proxyCmd)) } else if !hks.FindServer(KM_PROXY) { return nil, nil, fmt.Errorf("either run this scheduler via km or else --proxy-path is required") } else if s.ExecutorPath != "" { return nil, nil, fmt.Errorf("proxy can only use km binary if executor does the same") } // else, executor is smart enough to know when proxy-path is required, or to use km //TODO(jdef): provide some way (env var?) for users to customize executor config //TODO(jdef): set -address to 127.0.0.1 if `address` is 127.0.0.1 //TODO(jdef): propagate dockercfg from RootDirectory? apiServerArgs := strings.Join(s.APIServerList, ",") ci.Arguments = append(ci.Arguments, fmt.Sprintf("--api-servers=%s", apiServerArgs)) ci.Arguments = append(ci.Arguments, fmt.Sprintf("--v=%d", s.ExecutorLogV)) ci.Arguments = append(ci.Arguments, fmt.Sprintf("--allow-privileged=%t", s.AllowPrivileged)) ci.Arguments = append(ci.Arguments, fmt.Sprintf("--suicide-timeout=%v", s.ExecutorSuicideTimeout)) if s.ExecutorBindall { //TODO(jdef) determine whether hostname-override is really needed for bindall because //it conflicts with qinglet node status checks/updates //ci.Arguments = append(ci.Arguments, "--hostname-override=0.0.0.0") ci.Arguments = append(ci.Arguments, "--address=0.0.0.0") } ci.Arguments = append(ci.Arguments, fmt.Sprintf("--proxy-bindall=%v", s.ExecutorProxyBindall)) ci.Arguments = append(ci.Arguments, fmt.Sprintf("--run-proxy=%v", s.ExecutorRunProxy)) ci.Arguments = append(ci.Arguments, fmt.Sprintf("--cadvisor-port=%v", s.QingletCadvisorPort)) ci.Arguments = append(ci.Arguments, fmt.Sprintf("--sync-frequency=%v", s.QingletSyncFrequency)) if s.AuthPath != "" { //TODO(jdef) should probably support non-local files, e.g. hdfs:///some/config/file uri, basename := s.serveFrameworkArtifact(s.AuthPath) ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri)}) ci.Arguments = append(ci.Arguments, fmt.Sprintf("--auth-path=%s", basename)) } appendOptional := func(name string, value string) { if value != "" { ci.Arguments = append(ci.Arguments, fmt.Sprintf("--%s=%s", name, value)) } } if s.ClusterDNS != nil { appendOptional("cluster-dns", s.ClusterDNS.String()) } appendOptional("cluster-domain", s.ClusterDomain) appendOptional("root-dir", s.QingletRootDirectory) appendOptional("docker-endpoint", s.QingletDockerEndpoint) appendOptional("pod-infra-container-image", s.QingletPodInfraContainerImage) appendOptional("host-network-sources", s.QingletHostNetworkSources) appendOptional("network-plugin", s.QingletNetworkPluginName) log.V(1).Infof("prepared executor command %q with args '%+v'", ci.GetValue(), ci.Arguments) // Create mesos scheduler driver. info := &mesos.ExecutorInfo{ Command: ci, Name: proto.String(execcfg.DefaultInfoName), Source: proto.String(execcfg.DefaultInfoSource), } // Check for staticPods if s.StaticPodsConfigPath != "" { bs, numberStaticPods, err := archive.ZipDir(s.StaticPodsConfigPath) if err != nil { return nil, nil, err } info.Data = bs // Adjust the resource accounting for the executor. // Currently each podTask accounts the default amount of resources. // TODO(joerg84) adapt to actual resources specified by pods. log.Infof("Detected %d staticPods in Configuration.", numberStaticPods) info.Resources = []*mesos.Resource{ mutil.NewScalarResource("cpus", float64(numberStaticPods)*podtask.DefaultContainerCpus), mutil.NewScalarResource("mem", float64(numberStaticPods)*podtask.DefaultContainerMem), } } // calculate ExecutorInfo hash to be used for validating compatibility // of ExecutorInfo's generated by other HA schedulers. ehash := hashExecutorInfo(info) eid := uid.New(ehash, execcfg.DefaultInfoID) info.ExecutorId = &mesos.ExecutorID{Value: proto.String(eid.String())} return info, eid, nil }