// poll cluster-health until result
func getClusterHealth(m platform.Machine, csize int) error {
var err error
var b []byte
checker := func() error {
b, err := m.SSH("etcdctl cluster-health")
if err != nil {
return err
}
// repsonse should include "healthy" for each machine and for cluster
if strings.Count(string(b), "healthy") != (csize*2)+1 {
return fmt.Errorf("unexpected etcdctl output")
}
plog.Infof("cluster healthy")
return nil
}
err = util.Retry(15, 10*time.Second, checker)
if err != nil {
return fmt.Errorf("health polling failed: %v: %s", err, b)
}
return nil
}
func checkEtcdVersion(cluster platform.Cluster, m platform.Machine, expected string) error {
var b []byte
var err error
checker := func() error {
cmd := cluster.NewCommand("curl", "-L", fmt.Sprintf("http://%v:2379/version", m.IP()))
b, err = cmd.Output()
if err != nil {
return fmt.Errorf("curl failed: %v", err)
}
return nil
}
if err := util.Retry(15, 10*time.Second, checker); err != nil {
return err
}
plog.Infof("got version: %s", b)
if string(b) != expected {
return fmt.Errorf("expected %v, got %s", expected, b)
}
return nil
}
func checkEtcdVersion(cluster platform.Cluster, m platform.Machine, expected string) error {
var b []byte
checker := func() error {
out, err := m.SSH(fmt.Sprintf("curl -s -L http://%s:2379/version", m.IP()))
if err != nil {
return fmt.Errorf("curl failed: %v", out)
}
b = out
return nil
}
if err := util.Retry(15, 10*time.Second, checker); err != nil {
return err
}
plog.Infof("got version: %s", b)
if string(b) != expected {
return fmt.Errorf("expected %v, got %s", expected, b)
}
return nil
}
// ping sends icmp packets from machine a to b using the ping tool.
func ping(a, b platform.Machine, ifname string) error {
srcip, err := mach2bip(a, ifname)
if err != nil {
return fmt.Errorf("failed to get docker bridge ip: %v", err)
}
dstip, err := mach2bip(b, ifname)
if err != nil {
return fmt.Errorf("failed to get docker bridge ip: %v", err)
}
// ensure the docker bridges have the right network
_, ipnet, _ := net.ParseCIDR("10.254.0.0/16")
if !ipnet.Contains(net.ParseIP(srcip)) || !ipnet.Contains(net.ParseIP(dstip)) {
return fmt.Errorf("bridge ips (%s %s) not in flannel network (%s)", srcip, dstip, ipnet)
}
plog.Infof("ping from %s(%s) to %s(%s)", a.ID(), srcip, b.ID(), dstip)
cmd := fmt.Sprintf("ping -c 10 -I %s %s", srcip, dstip)
out, err := a.SSH(cmd)
if err != nil {
return fmt.Errorf("ping from %s to %s failed: %s: %v", a.ID(), b.ID(), out, err)
}
return nil
}
// poll cluster-health until result
func getClusterHealth(m platform.Machine, csize int) error {
const (
retries = 5
retryWait = 3 * time.Second
)
var err error
var b []byte
for i := 0; i < retries; i++ {
b, err = m.SSH("etcdctl cluster-health")
if err != nil {
plog.Debugf("retrying health check, hit failure %v", err)
time.Sleep(retryWait)
continue
}
// repsonse should include "healthy" for each machine and for cluster
if strings.Count(string(b), "healthy") == csize+1 {
plog.Infof("cluster healthy")
return nil
}
}
if err != nil {
return fmt.Errorf("health polling failed: %v: %s", err, b)
} else {
return fmt.Errorf("status unhealthy or incomplete: %s", b)
}
}
func checkEtcdVersion(cluster platform.Cluster, m platform.Machine, expected string) error {
const (
retries = 5
retryWait = 3 * time.Second
)
var err error
var b []byte
for i := 0; i < retries; i++ {
cmd := cluster.NewCommand("curl", "-L", fmt.Sprintf("http://%v:2379/version", m.IP()))
b, err = cmd.Output()
if err != nil {
plog.Infof("retrying version check, hit failure %v", err)
time.Sleep(retryWait)
continue
}
break
}
if err != nil {
return fmt.Errorf("curling version: %v", err)
}
plog.Infof("got version: %s", b)
if string(b) != expected {
return fmt.Errorf("expected %v, got %s", expected, b)
}
return nil
}
// run etcd on each cluster machine
func startEtcd2(m platform.Machine) error {
etcdStart := "sudo systemctl start etcd2.service"
_, err := m.SSH(etcdStart)
if err != nil {
return fmt.Errorf("start etcd2.service on %v failed: %s", m.IP(), err)
}
return nil
}
// stop etcd on each cluster machine
func stopEtcd2(m platform.Machine) error {
// start etcd instance
etcdStop := "sudo systemctl stop etcd2.service"
_, err := m.SSH(etcdStop)
if err != nil {
return fmt.Errorf("stop etcd.2service on failed: %s", err)
}
return nil
}
// Run and configure the coreos-kubernetes generic install scripts.
func runInstallScript(m platform.Machine, script string, options map[string]string) error {
// use built-in kubelet-wrapper if on-disk file does not exist
// on-disk wrapper should exist as of release 960.0.0
if _, err := m.SSH("sudo stat /usr/lib/coreos/kubelet-wrapper"); err != nil {
plog.Errorf("on-disk kubelet-wrapper not found, using test's built-in version")
options["KUBELET_PATH"] = "/home/core/rktkube.sh"
}
var buffer = new(bytes.Buffer)
tmpl, err := template.New("installScript").Parse(script)
if err != nil {
return err
}
if err := tmpl.Execute(buffer, options); err != nil {
return err
}
if err := platform.InstallFile(buffer, m, "/home/core/install.sh"); err != nil {
return err
}
// only used if kubelet-wrapper doesn't exist
in := strings.NewReader(rktkube)
if err := platform.InstallFile(in, m, "/home/core/rktkube.sh"); err != nil {
return err
}
// use client to collect stderr
client, err := m.SSHClient()
if err != nil {
return err
}
defer client.Close()
session, err := client.NewSession()
if err != nil {
return err
}
defer session.Close()
stderr := bytes.NewBuffer(nil)
session.Stderr = stderr
err = session.Start("sudo /home/core/install.sh")
if err != nil {
return err
}
if err := session.Wait(); err != nil {
return fmt.Errorf("%s", stderr)
}
return nil
}
// get docker bridge ip from a machine
func mach2bip(m platform.Machine, ifname string) (string, error) {
// note the escaped % in awk.
out, err := m.SSH(fmt.Sprintf(`ip -4 -o addr show dev %s primary | awk -F " +|/" '{printf "%%s", $4}'`, ifname))
if err != nil {
return "", err
}
// XXX(mischief): unfortunately `ip` does not return a nonzero status if the interface doesn't exist.
if len(out) == 0 {
return "", fmt.Errorf("interface %q doesn't exist?", ifname)
}
return string(out), nil
}
// checkUsrPartition inspects /proc/cmdline of the machine, looking for the
// expected partition mounted at /usr.
func checkUsrPartition(m platform.Machine, accept []string) error {
out, err := m.SSH("cat /proc/cmdline")
if err != nil {
return fmt.Errorf("cat /proc/cmdline: %v: %v", out, err)
}
vars := splitSpaceEnv(string(out))
for _, a := range accept {
if vars["mount.usr"] == a {
return nil
}
}
return fmt.Errorf("mount.usr not one of %q", strings.Join(accept, " "))
}
func generateMasterTLSAssets(master platform.Machine, options map[string]string) error {
var buffer = new(bytes.Buffer)
tmpl, err := template.New("masterCNF").Parse(masterCNF)
if err != nil {
return err
}
if err := tmpl.Execute(buffer, options); err != nil {
return err
}
if err := platform.InstallFile(buffer, master, "/home/core/openssl.cnf"); err != nil {
return err
}
var cmds = []string{
// gen master assets
"openssl genrsa -out ca-key.pem 2048",
`openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca"`,
"openssl genrsa -out apiserver-key.pem 2048",
`openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config openssl.cnf`,
"openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf",
// gen cluster admin keypair
"openssl genrsa -out admin-key.pem 2048",
`openssl req -new -key admin-key.pem -out admin.csr -subj "/CN=kube-admin"`,
"openssl x509 -req -in admin.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin.pem -days 365",
// move into /etc/kubernetes/ssl
"sudo mkdir -p /etc/kubernetes/ssl",
"sudo cp /home/core/ca.pem /etc/kubernetes/ssl/ca.pem",
"sudo cp /home/core/apiserver.pem /etc/kubernetes/ssl/apiserver.pem",
"sudo cp /home/core/apiserver-key.pem /etc/kubernetes/ssl/apiserver-key.pem",
"sudo chmod 600 /etc/kubernetes/ssl/*-key.pem",
"sudo chown root:root /etc/kubernetes/ssl/*-key.pem",
}
for _, cmd := range cmds {
b, err := master.SSH(cmd)
if err != nil {
return fmt.Errorf("Failed on cmd: %s with error: %s and output %s", cmd, err, b)
}
}
return nil
}
// replace default binary for etcd2.service with given binary
func replaceEtcd2Bin(m platform.Machine, newPath string) error {
if !filepath.IsAbs(newPath) {
return fmt.Errorf("newPath must be an absolute filepath")
}
override := "\"[Service]\nExecStart=\nExecStart=" + newPath
override += "\nEnvironment=ETCD_SNAPSHOT_COUNT=10" + "\"" // make it easy to trigger snapshots
_, err := m.SSH(fmt.Sprintf("echo %v | sudo tee /run/systemd/system/etcd2.service.d/99-exec.conf", override))
if err != nil {
return err
}
_, err = m.SSH("sudo systemctl daemon-reload")
if err != nil {
return err
}
return nil
}
func secretCheck(master platform.Machine, nodes []platform.Machine) error {
// create yaml files
secret := strings.NewReader(secretYAML)
pod := strings.NewReader(secretPodYAML)
if err := platform.InstallFile(secret, master, "./secret.yaml"); err != nil {
return err
}
if err := platform.InstallFile(pod, master, "./secret-pod.yaml"); err != nil {
return err
}
if _, err := master.SSH("./kubectl create -f secret.yaml"); err != nil {
return err
}
_, err := master.SSH("./kubectl describe secret test-secret")
if err != nil {
return err
}
b, err := master.SSH("./kubectl create -f secret-pod.yaml")
if err != nil {
return err
}
expectedOutput := "value-1"
if strings.Contains(strings.TrimSpace(string(b)), expectedOutput) {
return fmt.Errorf("error detecting secret pod")
}
return nil
}
func nginxCheck(master platform.Machine, nodes []platform.Machine) error {
pod := strings.NewReader(nginxPodYAML)
if err := platform.InstallFile(pod, master, "./nginx-pod.yaml"); err != nil {
return err
}
if _, err := master.SSH("./kubectl create -f nginx-pod.yaml"); err != nil {
return err
}
// wait for pod status to be 'Running'
podIsRunning := func() error {
b, err := master.SSH("./kubectl get pod nginx -o=template -t={{.status.phase}}")
if err != nil {
return err
}
if !bytes.Equal(b, []byte("Running")) {
return fmt.Errorf("nginx pod not running: %s", b)
}
return nil
}
if err := util.Retry(10, 5*time.Second, podIsRunning); err != nil {
return err
}
// delete pod
_, err := master.SSH("./kubectl delete pods nginx")
if err != nil {
return err
}
return nil
}
func nodeCheck(master platform.Machine, nodes []platform.Machine) error {
b, err := master.SSH("./kubectl get nodes")
if err != nil {
return err
}
// parse kubectl output for IPs
addrMap := map[string]struct{}{}
for _, line := range strings.Split(string(b), "\n")[1:] {
addrMap[strings.SplitN(line, " ", 2)[0]] = struct{}{}
}
if len(addrMap) != len(nodes) {
return fmt.Errorf("cannot detect all nodes in kubectl output \n%v\n%v", addrMap, nodes)
}
for _, node := range nodes {
if _, ok := addrMap[node.PrivateIP()]; !ok {
return fmt.Errorf("node IP missing from kubectl get nodes")
}
}
return nil
}
func doStart(m platform.Machine, version int, block bool) error {
// start etcd instance
var etcdStart string
if version == 1 {
etcdStart = "sudo systemctl start etcd.service"
} else if version == 2 {
etcdStart = "sudo systemctl start etcd2.service"
} else {
return fmt.Errorf("etcd version unspecified")
}
if !block {
etcdStart += " --no-block"
}
_, err := m.SSH(etcdStart)
if err != nil {
return fmt.Errorf("SSH cmd to %v failed: %s", m.IP(), err)
}
return nil
}
func copyFile(mach platform.Machine, r io.Reader, path string) error {
sess, err := mach.SSHSession()
if err != nil {
return err
}
defer sess.Close()
wr, err := sess.StdinPipe()
if err != nil {
return err
}
if err := sess.Start(fmt.Sprintf("tee %s", path)); err != nil {
return err
}
go func() {
io.Copy(wr, r)
wr.Close()
}()
return sess.Wait()
}
// https://coreos.com/kubernetes/docs/latest/configure-kubectl.html
func configureKubectl(m platform.Machine, server string, version string) error {
// ignore suffix like '-coreos.1' on version to grab upstream
semverPrefix := regexp.MustCompile(`^v[\d]+\.[\d]+\.[\d]+`)
version = semverPrefix.FindString(version)
if version == "" {
return fmt.Errorf("Failure to parse kubectl version")
}
var (
ca = "/home/core/ca.pem"
adminKey = "/home/core/admin-key.pem"
adminCert = "/home/core/admin.pem"
kubeURL = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%v/bin/linux/amd64/kubectl", version)
)
if _, err := m.SSH("wget -q " + kubeURL); err != nil {
return err
}
if _, err := m.SSH("chmod +x ./kubectl"); err != nil {
return err
}
// cmds to configure kubectl
cmds := []string{
fmt.Sprintf("./kubectl config set-cluster default-cluster --server=https://%v --certificate-authority=%v", server, ca),
fmt.Sprintf("./kubectl config set-credentials default-admin --certificate-authority=%v --client-key=%v --client-certificate=%v", ca, adminKey, adminCert),
"./kubectl config set-context default-system --cluster=default-cluster --user=default-admin",
"./kubectl config use-context default-system",
}
for _, cmd := range cmds {
b, err := m.SSH(cmd)
if err != nil {
return fmt.Errorf("Failed on cmd: %s with error: %s and output %s", cmd, err, b)
}
}
return nil
}