// startMasterOrDie starts a qingyuan master and an httpserver to handle api requests
func startMasterOrDie(masterConfig *master.Config) (*master.Master, *httptest.Server, *tools.EtcdHelper) {
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
var helper tools.EtcdHelper
var err error
if masterConfig == nil {
helper, err = master.NewEtcdHelper(NewEtcdClient(), "", etcdtest.PathPrefix())
if err != nil {
glog.Fatalf("Failed to create etcd helper for master %v", err)
}
masterConfig = &master.Config{
EtcdHelper: helper,
QingletClient: client.FakeQingletClient{},
EnableLogsSupport: false,
EnableProfiling: true,
EnableUISupport: false,
APIPrefix: "/api",
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admit.NewAlwaysAdmit(),
}
} else {
helper = masterConfig.EtcdHelper
}
m = master.New(masterConfig)
return m, s, &helper
}
// RunApiServer starts an API server in a go routine.
func runApiServer(etcdClient tools.EtcdClient, addr net.IP, port int, masterServiceNamespace string) {
handler := delegateHandler{}
helper, err := master.NewEtcdHelper(etcdClient, "", master.DefaultEtcdPathPrefix)
if err != nil {
glog.Fatalf("Unable to get etcd helper: %v", err)
}
// Create a master and install handlers into mux.
m := master.New(&master.Config{
EtcdHelper: helper,
QingletClient: &client.HTTPQingletClient{
Client: http.DefaultClient,
Port: 10250,
},
EnableCoreControllers: true,
EnableLogsSupport: false,
EnableSwaggerSupport: true,
EnableProfiling: *enableProfiling,
APIPrefix: "/api",
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
ReadWritePort: port,
PublicAddress: addr,
MasterServiceNamespace: masterServiceNamespace,
})
handler.delegate = m.InsecureHandler
go http.ListenAndServe(fmt.Sprintf("%s:%d", addr, port), &handler)
}
func newEtcd(etcdConfigFile string, etcdServerList util.StringList, storageVersion string, pathPrefix string) (helper tools.EtcdHelper, err error) {
var client tools.EtcdGetSet
if etcdConfigFile != "" {
client, err = etcd.NewClientFromFile(etcdConfigFile)
if err != nil {
return helper, err
}
} else {
etcdClient := etcd.NewClient(etcdServerList)
transport := &http.Transport{
Dial: forked.Dial,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
MaxIdleConnsPerHost: 500,
}
etcdClient.SetTransport(transport)
client = etcdClient
}
return master.NewEtcdHelper(client, storageVersion, pathPrefix)
}
// TODO: Merge this into startMasterOrDie.
func RunAMaster(t *testing.T) (*master.Master, *httptest.Server) {
helper, err := master.NewEtcdHelper(NewEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
m := master.New(&master.Config{
EtcdHelper: helper,
QingletClient: client.FakeQingletClient{},
EnableLogsSupport: false,
EnableProfiling: true,
EnableUISupport: false,
APIPrefix: "/api",
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admit.NewAlwaysAdmit(),
})
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
return m, s
}
func NewHelper() (tools.EtcdHelper, error) {
return master.NewEtcdHelper(NewEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
}
// startServiceAccountTestServer returns a started server
// It is the responsibility of the caller to ensure the returned stopFunc is called
func startServiceAccountTestServer(t *testing.T) (*client.Client, client.Config, func()) {
deleteAllEtcdKeys()
// Etcd
helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Listener
var m *master.Master
apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
// Anonymous client config
clientConfig := client.Config{Host: apiServer.URL, Version: testapi.Version()}
// Root client
rootClient := client.NewOrDie(&client.Config{Host: apiServer.URL, Version: testapi.Version(), BearerToken: rootToken})
// Set up two authenticators:
// 1. A token authenticator that maps the rootToken to the "root" user
// 2. A ServiceAccountToken authenticator that validates ServiceAccount tokens
rootTokenAuth := authenticator.TokenFunc(func(token string) (user.Info, bool, error) {
if token == rootToken {
return &user.DefaultInfo{rootUserName, "", []string{}}, true, nil
}
return nil, false, nil
})
serviceAccountKey, err := rsa.GenerateKey(rand.Reader, 2048)
serviceAccountTokenGetter := serviceaccount.NewGetterFromClient(rootClient)
serviceAccountTokenAuth := serviceaccount.JWTTokenAuthenticator([]*rsa.PublicKey{&serviceAccountKey.PublicKey}, true, serviceAccountTokenGetter)
authenticator := union.New(
bearertoken.New(rootTokenAuth),
bearertoken.New(serviceAccountTokenAuth),
)
// Set up a stub authorizer:
// 1. The "root" user is allowed to do anything
// 2. ServiceAccounts named "ro" are allowed read-only operations in their namespace
// 3. ServiceAccounts named "rw" are allowed any operation in their namespace
authorizer := authorizer.AuthorizerFunc(func(attrs authorizer.Attributes) error {
username := attrs.GetUserName()
ns := attrs.GetNamespace()
// If the user is "root"...
if username == rootUserName {
// allow them to do anything
return nil
}
// If the user is a service account...
if serviceAccountNamespace, serviceAccountName, err := serviceaccount.SplitUsername(username); err == nil {
// Limit them to their own namespace
if serviceAccountNamespace == ns {
switch serviceAccountName {
case readOnlyServiceAccountName:
if attrs.IsReadOnly() {
return nil
}
case readWriteServiceAccountName:
return nil
}
}
}
return fmt.Errorf("User %s is denied (ns=%s, readonly=%v, resource=%s)", username, ns, attrs.IsReadOnly(), attrs.GetResource())
})
// Set up admission plugin to auto-assign serviceaccounts to pods
serviceAccountAdmission := serviceaccountadmission.NewServiceAccount(rootClient)
// Create a master and install handlers into mux.
m = master.New(&master.Config{
EtcdHelper: helper,
QingletClient: client.FakeQingletClient{},
EnableLogsSupport: false,
EnableUISupport: false,
EnableIndex: true,
APIPrefix: "/api",
Authenticator: authenticator,
Authorizer: authorizer,
AdmissionControl: serviceAccountAdmission,
})
// Start the service account and service account token controllers
tokenController := serviceaccount.NewTokensController(rootClient, serviceaccount.DefaultTokenControllerOptions(serviceaccount.JWTTokenGenerator(serviceAccountKey)))
tokenController.Run()
serviceAccountController := serviceaccount.NewServiceAccountsController(rootClient, serviceaccount.DefaultServiceAccountsControllerOptions())
serviceAccountController.Run()
// Start the admission plugin reflectors
serviceAccountAdmission.Run()
stop := func() {
tokenController.Stop()
serviceAccountController.Stop()
serviceAccountAdmission.Stop()
apiServer.Close()
}
return rootClient, clientConfig, stop
}
func startComponents(firstManifestURL, secondManifestURL, apiVersion string) (string, string) {
// Setup
servers := []string{}
glog.Infof("Creating etcd client pointing to %v", servers)
handler := delegateHandler{}
apiServer := httptest.NewServer(&handler)
etcdClient := etcd.NewClient(servers)
sleep := 4 * time.Second
ok := false
for i := 0; i < 3; i++ {
keys, err := etcdClient.Get("/", false, false)
if err != nil {
glog.Warningf("Unable to list root etcd keys: %v", err)
if i < 2 {
time.Sleep(sleep)
sleep = sleep * sleep
}
continue
}
for _, node := range keys.Node.Nodes {
if _, err := etcdClient.Delete(node.Key, true); err != nil {
glog.Fatalf("Unable delete key: %v", err)
}
}
ok = true
break
}
if !ok {
glog.Fatalf("Failed to connect to etcd")
}
cl := client.NewOrDie(&client.Config{Host: apiServer.URL, Version: apiVersion})
helper, err := master.NewEtcdHelper(etcdClient, "", etcdtest.PathPrefix())
if err != nil {
glog.Fatalf("Unable to get etcd helper: %v", err)
}
// Master
host, port, err := net.SplitHostPort(strings.TrimLeft(apiServer.URL, "http://"))
if err != nil {
glog.Fatalf("Unable to parse URL '%v': %v", apiServer.URL, err)
}
portNumber, err := strconv.Atoi(port)
if err != nil {
glog.Fatalf("Nonnumeric port? %v", err)
}
publicAddress := net.ParseIP(host)
if publicAddress == nil {
glog.Fatalf("no public address for %s", host)
}
// Create a master and install handlers into mux.
m := master.New(&master.Config{
EtcdHelper: helper,
QingletClient: fakeQingletClient{},
EnableCoreControllers: true,
EnableLogsSupport: false,
EnableProfiling: true,
APIPrefix: "/api",
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admit.NewAlwaysAdmit(),
ReadWritePort: portNumber,
PublicAddress: publicAddress,
CacheTimeout: 2 * time.Second,
})
handler.delegate = m.Handler
// Scheduler
schedulerConfigFactory := factory.NewConfigFactory(cl)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
glog.Fatalf("Couldn't create scheduler config: %v", err)
}
eventBroadcaster := record.NewBroadcaster()
schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"})
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(cl.Events(""))
scheduler.New(schedulerConfig).Run()
endpoints := service.NewEndpointController(cl)
// ensure the service endpoints are sync'd several times within the window that the integration tests wait
go endpoints.Run(3, util.NeverStop)
controllerManager := replicationControllerPkg.NewReplicationManager(cl, replicationControllerPkg.BurstReplicas)
// TODO: Write an integration test for the replication controllers watch.
go controllerManager.Run(3, util.NeverStop)
nodeController := nodecontroller.NewNodeController(nil, cl, 10, 5*time.Minute, nodecontroller.NewPodEvictor(util.NewFakeRateLimiter()),
40*time.Second, 60*time.Second, 5*time.Second, nil, false)
nodeController.Run(5 * time.Second)
cadvisorInterface := new(cadvisor.Fake)
// Qinglet (localhost)
testRootDir := makeTempDirOrDie("qinglet_integ_1.", "")
configFilePath := makeTempDirOrDie("config", testRootDir)
glog.Infof("Using %s as root dir for qinglet #1", testRootDir)
fakeDocker1.VersionInfo = docker.Env{"ApiVersion=1.15"}
kcfg := qingletapp.SimpleQinglet(cl, &fakeDocker1, "localhost", testRootDir, firstManifestURL, "127.0.0.1", 10250, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, configFilePath, nil, qingcontainer.FakeOS{})
qingletapp.RunQinglet(kcfg, nil)
// Qinglet (machine)
// Create a second qinglet so that the guestbook example's two redis slaves both
// have a place they can schedule.
testRootDir = makeTempDirOrDie("qinglet_integ_2.", "")
glog.Infof("Using %s as root dir for qinglet #2", testRootDir)
fakeDocker2.VersionInfo = docker.Env{"ApiVersion=1.15"}
kcfg = qingletapp.SimpleQinglet(cl, &fakeDocker2, "127.0.0.1", testRootDir, secondManifestURL, "127.0.0.1", 10251, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, "", nil, qingcontainer.FakeOS{})
qingletapp.RunQinglet(kcfg, nil)
return apiServer.URL, configFilePath
}