func secretCheck(master platform.Machine, nodes []platform.Machine) error {
// create yaml files
secret := strings.NewReader(secretYAML)
pod := strings.NewReader(secretPodYAML)
if err := platform.InstallFile(secret, master, "./secret.yaml"); err != nil {
return err
}
if err := platform.InstallFile(pod, master, "./secret-pod.yaml"); err != nil {
return err
}
if _, err := master.SSH("./kubectl create -f secret.yaml"); err != nil {
return err
}
_, err := master.SSH("./kubectl describe secret test-secret")
if err != nil {
return err
}
b, err := master.SSH("./kubectl create -f secret-pod.yaml")
if err != nil {
return err
}
expectedOutput := "value-1"
if strings.Contains(strings.TrimSpace(string(b)), expectedOutput) {
return fmt.Errorf("error detecting secret pod")
}
return nil
}
// Run and configure the coreos-kubernetes generic install scripts.
func runInstallScript(m platform.Machine, script string, options map[string]string) error {
// use built-in kubelet-wrapper if on-disk file does not exist
// on-disk wrapper should exist as of release 960.0.0
if _, err := m.SSH("sudo stat /usr/lib/coreos/kubelet-wrapper"); err != nil {
plog.Errorf("on-disk kubelet-wrapper not found, using test's built-in version")
options["KUBELET_PATH"] = "/home/core/rktkube.sh"
}
var buffer = new(bytes.Buffer)
tmpl, err := template.New("installScript").Parse(script)
if err != nil {
return err
}
if err := tmpl.Execute(buffer, options); err != nil {
return err
}
if err := platform.InstallFile(buffer, m, "/home/core/install.sh"); err != nil {
return err
}
// only used if kubelet-wrapper doesn't exist
in := strings.NewReader(rktkube)
if err := platform.InstallFile(in, m, "/home/core/rktkube.sh"); err != nil {
return err
}
// use client to collect stderr
client, err := m.SSHClient()
if err != nil {
return err
}
defer client.Close()
session, err := client.NewSession()
if err != nil {
return err
}
defer session.Close()
stderr := bytes.NewBuffer(nil)
session.Stderr = stderr
err = session.Start("sudo /home/core/install.sh")
if err != nil {
return err
}
if err := session.Wait(); err != nil {
return fmt.Errorf("%s", stderr)
}
return nil
}
func nginxCheck(master platform.Machine, nodes []platform.Machine) error {
pod := strings.NewReader(nginxPodYAML)
if err := platform.InstallFile(pod, master, "./nginx-pod.yaml"); err != nil {
return err
}
if _, err := master.SSH("./kubectl create -f nginx-pod.yaml"); err != nil {
return err
}
// wait for pod status to be 'Running'
podIsRunning := func() error {
b, err := master.SSH("./kubectl get pod nginx -o=template -t={{.status.phase}}")
if err != nil {
return err
}
if !bytes.Equal(b, []byte("Running")) {
return fmt.Errorf("nginx pod not running: %s", b)
}
return nil
}
if err := util.Retry(10, 5*time.Second, podIsRunning); err != nil {
return err
}
// delete pod
_, err := master.SSH("./kubectl delete pods nginx")
if err != nil {
return err
}
return nil
}
// Test fleet running through an etcd2 proxy.
func Proxy(c platform.TestCluster) error {
masterconf.CoreOS.Etcd2.Discovery, _ = c.GetDiscoveryURL(1)
master, err := c.NewMachine(masterconf.String())
if err != nil {
return fmt.Errorf("Cluster.NewMachine: %s", err)
}
defer master.Destroy()
proxyconf.CoreOS.Etcd2.Discovery = masterconf.CoreOS.Etcd2.Discovery
proxy, err := c.NewMachine(proxyconf.String())
if err != nil {
return fmt.Errorf("Cluster.NewMachine: %s", err)
}
defer proxy.Destroy()
err = platform.InstallFile(strings.NewReader(fleetunit), proxy, "/home/core/hello.service")
if err != nil {
return fmt.Errorf("InstallFile: %s", err)
}
// settling...
fleetStart := func() error {
_, err = proxy.SSH("fleetctl start /home/core/hello.service")
if err != nil {
return fmt.Errorf("fleetctl start: %s", err)
}
return nil
}
if err := util.Retry(5, 5*time.Second, fleetStart); err != nil {
return fmt.Errorf("fleetctl start failed: %v", err)
}
var status []byte
fleetList := func() error {
status, err = proxy.SSH("fleetctl list-units -l -fields active -no-legend")
if err != nil {
return fmt.Errorf("fleetctl list-units: %s", err)
}
if !bytes.Equal(status, []byte("active")) {
return fmt.Errorf("unit not active")
}
return nil
}
if err := util.Retry(5, 1*time.Second, fleetList); err != nil {
return fmt.Errorf("fleetctl list-units failed: %v", err)
}
return nil
}
func generateMasterTLSAssets(master platform.Machine, options map[string]string) error {
var buffer = new(bytes.Buffer)
tmpl, err := template.New("masterCNF").Parse(masterCNF)
if err != nil {
return err
}
if err := tmpl.Execute(buffer, options); err != nil {
return err
}
if err := platform.InstallFile(buffer, master, "/home/core/openssl.cnf"); err != nil {
return err
}
var cmds = []string{
// gen master assets
"openssl genrsa -out ca-key.pem 2048",
`openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca"`,
"openssl genrsa -out apiserver-key.pem 2048",
`openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config openssl.cnf`,
"openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf",
// gen cluster admin keypair
"openssl genrsa -out admin-key.pem 2048",
`openssl req -new -key admin-key.pem -out admin.csr -subj "/CN=kube-admin"`,
"openssl x509 -req -in admin.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin.pem -days 365",
// move into /etc/kubernetes/ssl
"sudo mkdir -p /etc/kubernetes/ssl",
"sudo cp /home/core/ca.pem /etc/kubernetes/ssl/ca.pem",
"sudo cp /home/core/apiserver.pem /etc/kubernetes/ssl/apiserver.pem",
"sudo cp /home/core/apiserver-key.pem /etc/kubernetes/ssl/apiserver-key.pem",
"sudo chmod 600 /etc/kubernetes/ssl/*-key.pem",
"sudo chown root:root /etc/kubernetes/ssl/*-key.pem",
}
for _, cmd := range cmds {
b, err := master.SSH(cmd)
if err != nil {
return fmt.Errorf("Failed on cmd: %s with error: %s and output %s", cmd, err, b)
}
}
return nil
}
func generateWorkerTLSAssets(master platform.Machine, workers []platform.Machine) error {
for i, worker := range workers {
// copy tls assets from master to workers
err := platform.TransferFile(master, "/etc/kubernetes/ssl/ca.pem", worker, "/home/core/ca.pem")
if err != nil {
return err
}
err = platform.TransferFile(master, "/home/core/ca-key.pem", worker, "/home/core/ca-key.pem")
if err != nil {
return err
}
// place worker-openssl.cnf on workers
cnf := strings.Replace(workerCNF, "{{.WORKER_IP}}", worker.PrivateIP(), -1)
in := strings.NewReader(cnf)
if err := platform.InstallFile(in, worker, "/home/core/worker-openssl.cnf"); err != nil {
return err
}
// gen certs
workerFQDN := fmt.Sprintf("kube-worker-%v", i)
cmds := []string{
fmt.Sprintf("openssl genrsa -out worker-key.pem 2048"),
fmt.Sprintf(`openssl req -new -key worker-key.pem -out %v-worker.csr -subj "/CN=%v" -config worker-openssl.cnf`, workerFQDN, workerFQDN),
fmt.Sprintf(`openssl x509 -req -in %v-worker.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out worker.pem -days 365 -extensions v3_req -extfile worker-openssl.cnf`, workerFQDN),
// move into /etc/kubernetes/ssl
"sudo mkdir -p /etc/kubernetes/ssl",
"sudo chmod 600 /home/core/*-key.pem",
"sudo chown root:root /home/core/*-key.pem",
"sudo cp /home/core/worker.pem /etc/kubernetes/ssl/worker.pem",
"sudo cp /home/core/worker-key.pem /etc/kubernetes/ssl/worker-key.pem",
"sudo cp /home/core/ca.pem /etc/kubernetes/ssl/ca.pem",
}
for _, cmd := range cmds {
b, err := worker.SSH(cmd)
if err != nil {
return fmt.Errorf("Failed on cmd: %s with error: %s and output %s", cmd, err, b)
}
}
}
return nil
}