Example #1
0
// poll cluster-health until result
func getClusterHealth(m platform.Machine, csize int) error {
	var err error
	var b []byte

	checker := func() error {
		b, err := m.SSH("etcdctl cluster-health")
		if err != nil {
			return err
		}

		// repsonse should include "healthy" for each machine and for cluster
		if strings.Count(string(b), "healthy") != (csize*2)+1 {
			return fmt.Errorf("unexpected etcdctl output")
		}

		plog.Infof("cluster healthy")
		return nil
	}

	err = util.Retry(15, 10*time.Second, checker)
	if err != nil {
		return fmt.Errorf("health polling failed: %v: %s", err, b)
	}

	return nil
}
Example #2
0
func checkEtcdVersion(cluster platform.Cluster, m platform.Machine, expected string) error {
	var b []byte
	var err error

	checker := func() error {
		cmd := cluster.NewCommand("curl", "-L", fmt.Sprintf("http://%v:2379/version", m.IP()))
		b, err = cmd.Output()
		if err != nil {
			return fmt.Errorf("curl failed: %v", err)
		}

		return nil
	}

	if err := util.Retry(15, 10*time.Second, checker); err != nil {
		return err
	}

	plog.Infof("got version: %s", b)

	if string(b) != expected {
		return fmt.Errorf("expected %v, got %s", expected, b)
	}

	return nil
}
Example #3
0
func checkEtcdVersion(cluster platform.Cluster, m platform.Machine, expected string) error {
	var b []byte

	checker := func() error {
		out, err := m.SSH(fmt.Sprintf("curl -s -L http://%s:2379/version", m.IP()))
		if err != nil {
			return fmt.Errorf("curl failed: %v", out)
		}

		b = out

		return nil
	}

	if err := util.Retry(15, 10*time.Second, checker); err != nil {
		return err
	}

	plog.Infof("got version: %s", b)

	if string(b) != expected {
		return fmt.Errorf("expected %v, got %s", expected, b)
	}

	return nil
}
Example #4
0
// ping sends icmp packets from machine a to b using the ping tool.
func ping(a, b platform.Machine, ifname string) error {
	srcip, err := mach2bip(a, ifname)
	if err != nil {
		return fmt.Errorf("failed to get docker bridge ip: %v", err)
	}

	dstip, err := mach2bip(b, ifname)
	if err != nil {
		return fmt.Errorf("failed to get docker bridge ip: %v", err)
	}

	// ensure the docker bridges have the right network
	_, ipnet, _ := net.ParseCIDR("10.254.0.0/16")
	if !ipnet.Contains(net.ParseIP(srcip)) || !ipnet.Contains(net.ParseIP(dstip)) {
		return fmt.Errorf("bridge ips (%s %s) not in flannel network (%s)", srcip, dstip, ipnet)
	}

	plog.Infof("ping from %s(%s) to %s(%s)", a.ID(), srcip, b.ID(), dstip)

	cmd := fmt.Sprintf("ping -c 10 -I %s %s", srcip, dstip)
	out, err := a.SSH(cmd)
	if err != nil {
		return fmt.Errorf("ping from %s to %s failed: %s: %v", a.ID(), b.ID(), out, err)
	}

	return nil
}
Example #5
0
// poll cluster-health until result
func getClusterHealth(m platform.Machine, csize int) error {
	const (
		retries   = 5
		retryWait = 3 * time.Second
	)
	var err error
	var b []byte

	for i := 0; i < retries; i++ {
		b, err = m.SSH("etcdctl cluster-health")
		if err != nil {
			plog.Debugf("retrying health check, hit failure %v", err)
			time.Sleep(retryWait)
			continue
		}

		// repsonse should include "healthy" for each machine and for cluster
		if strings.Count(string(b), "healthy") == csize+1 {
			plog.Infof("cluster healthy")
			return nil
		}
	}

	if err != nil {
		return fmt.Errorf("health polling failed: %v: %s", err, b)
	} else {
		return fmt.Errorf("status unhealthy or incomplete: %s", b)
	}
}
Example #6
0
func checkEtcdVersion(cluster platform.Cluster, m platform.Machine, expected string) error {
	const (
		retries   = 5
		retryWait = 3 * time.Second
	)
	var err error
	var b []byte

	for i := 0; i < retries; i++ {
		cmd := cluster.NewCommand("curl", "-L", fmt.Sprintf("http://%v:2379/version", m.IP()))
		b, err = cmd.Output()
		if err != nil {
			plog.Infof("retrying version check, hit failure %v", err)
			time.Sleep(retryWait)
			continue
		}
		break
	}
	if err != nil {
		return fmt.Errorf("curling version: %v", err)
	}

	plog.Infof("got version: %s", b)

	if string(b) != expected {
		return fmt.Errorf("expected %v, got %s", expected, b)
	}
	return nil
}
Example #7
0
// run etcd on each cluster machine
func startEtcd2(m platform.Machine) error {
	etcdStart := "sudo systemctl start etcd2.service"
	_, err := m.SSH(etcdStart)
	if err != nil {
		return fmt.Errorf("start etcd2.service on %v failed: %s", m.IP(), err)
	}
	return nil
}
Example #8
0
// stop etcd on each cluster machine
func stopEtcd2(m platform.Machine) error {
	// start etcd instance
	etcdStop := "sudo systemctl stop etcd2.service"
	_, err := m.SSH(etcdStop)
	if err != nil {
		return fmt.Errorf("stop etcd.2service on failed: %s", err)
	}
	return nil
}
Example #9
0
// Run and configure the coreos-kubernetes generic install scripts.
func runInstallScript(m platform.Machine, script string, options map[string]string) error {
	// use built-in kubelet-wrapper if on-disk file does not exist
	// on-disk wrapper should exist as of release 960.0.0
	if _, err := m.SSH("sudo stat /usr/lib/coreos/kubelet-wrapper"); err != nil {
		plog.Errorf("on-disk kubelet-wrapper not found, using test's built-in version")
		options["KUBELET_PATH"] = "/home/core/rktkube.sh"
	}

	var buffer = new(bytes.Buffer)

	tmpl, err := template.New("installScript").Parse(script)
	if err != nil {
		return err
	}
	if err := tmpl.Execute(buffer, options); err != nil {
		return err
	}

	if err := platform.InstallFile(buffer, m, "/home/core/install.sh"); err != nil {
		return err
	}

	// only used if kubelet-wrapper doesn't exist
	in := strings.NewReader(rktkube)
	if err := platform.InstallFile(in, m, "/home/core/rktkube.sh"); err != nil {
		return err
	}

	// use client to collect stderr
	client, err := m.SSHClient()
	if err != nil {
		return err
	}
	defer client.Close()
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()

	stderr := bytes.NewBuffer(nil)
	session.Stderr = stderr

	err = session.Start("sudo /home/core/install.sh")
	if err != nil {
		return err
	}

	if err := session.Wait(); err != nil {
		return fmt.Errorf("%s", stderr)
	}

	return nil
}
Example #10
0
// get docker bridge ip from a machine
func mach2bip(m platform.Machine, ifname string) (string, error) {
	// note the escaped % in awk.
	out, err := m.SSH(fmt.Sprintf(`ip -4 -o addr show dev %s primary | awk -F " +|/" '{printf "%%s", $4}'`, ifname))
	if err != nil {
		return "", err
	}

	// XXX(mischief): unfortunately `ip` does not return a nonzero status if the interface doesn't exist.
	if len(out) == 0 {
		return "", fmt.Errorf("interface %q doesn't exist?", ifname)
	}

	return string(out), nil
}
Example #11
0
// checkUsrPartition inspects /proc/cmdline of the machine, looking for the
// expected partition mounted at /usr.
func checkUsrPartition(m platform.Machine, accept []string) error {
	out, err := m.SSH("cat /proc/cmdline")
	if err != nil {
		return fmt.Errorf("cat /proc/cmdline: %v: %v", out, err)
	}

	vars := splitSpaceEnv(string(out))
	for _, a := range accept {
		if vars["mount.usr"] == a {
			return nil
		}
	}

	return fmt.Errorf("mount.usr not one of %q", strings.Join(accept, " "))
}
Example #12
0
func generateMasterTLSAssets(master platform.Machine, options map[string]string) error {
	var buffer = new(bytes.Buffer)

	tmpl, err := template.New("masterCNF").Parse(masterCNF)
	if err != nil {
		return err
	}
	if err := tmpl.Execute(buffer, options); err != nil {
		return err
	}

	if err := platform.InstallFile(buffer, master, "/home/core/openssl.cnf"); err != nil {
		return err
	}

	var cmds = []string{
		// gen master assets
		"openssl genrsa -out ca-key.pem 2048",
		`openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca"`,
		"openssl genrsa -out apiserver-key.pem 2048",
		`openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config openssl.cnf`,
		"openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf",

		// gen cluster admin keypair
		"openssl genrsa -out admin-key.pem 2048",
		`openssl req -new -key admin-key.pem -out admin.csr -subj "/CN=kube-admin"`,
		"openssl x509 -req -in admin.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin.pem -days 365",

		// move into /etc/kubernetes/ssl
		"sudo mkdir -p /etc/kubernetes/ssl",
		"sudo cp /home/core/ca.pem /etc/kubernetes/ssl/ca.pem",
		"sudo cp /home/core/apiserver.pem /etc/kubernetes/ssl/apiserver.pem",
		"sudo cp /home/core/apiserver-key.pem /etc/kubernetes/ssl/apiserver-key.pem",
		"sudo chmod 600 /etc/kubernetes/ssl/*-key.pem",
		"sudo chown root:root /etc/kubernetes/ssl/*-key.pem",
	}

	for _, cmd := range cmds {
		b, err := master.SSH(cmd)
		if err != nil {
			return fmt.Errorf("Failed on cmd: %s with error: %s and output %s", cmd, err, b)
		}
	}
	return nil
}
Example #13
0
// replace default binary for etcd2.service with given binary
func replaceEtcd2Bin(m platform.Machine, newPath string) error {
	if !filepath.IsAbs(newPath) {
		return fmt.Errorf("newPath must be an absolute filepath")
	}

	override := "\"[Service]\nExecStart=\nExecStart=" + newPath
	override += "\nEnvironment=ETCD_SNAPSHOT_COUNT=10" + "\"" // make it easy to trigger snapshots

	_, err := m.SSH(fmt.Sprintf("echo %v | sudo tee /run/systemd/system/etcd2.service.d/99-exec.conf", override))
	if err != nil {
		return err
	}
	_, err = m.SSH("sudo systemctl daemon-reload")
	if err != nil {
		return err
	}
	return nil
}
Example #14
0
func secretCheck(master platform.Machine, nodes []platform.Machine) error {
	// create yaml files
	secret := strings.NewReader(secretYAML)
	pod := strings.NewReader(secretPodYAML)
	if err := platform.InstallFile(secret, master, "./secret.yaml"); err != nil {
		return err
	}
	if err := platform.InstallFile(pod, master, "./secret-pod.yaml"); err != nil {
		return err
	}

	if _, err := master.SSH("./kubectl create -f secret.yaml"); err != nil {
		return err
	}
	_, err := master.SSH("./kubectl describe secret test-secret")
	if err != nil {
		return err
	}

	b, err := master.SSH("./kubectl create -f secret-pod.yaml")
	if err != nil {
		return err
	}
	expectedOutput := "value-1"
	if strings.Contains(strings.TrimSpace(string(b)), expectedOutput) {
		return fmt.Errorf("error detecting secret pod")
	}

	return nil
}
Example #15
0
func nginxCheck(master platform.Machine, nodes []platform.Machine) error {
	pod := strings.NewReader(nginxPodYAML)
	if err := platform.InstallFile(pod, master, "./nginx-pod.yaml"); err != nil {
		return err
	}
	if _, err := master.SSH("./kubectl create -f nginx-pod.yaml"); err != nil {
		return err
	}
	// wait for pod status to be 'Running'
	podIsRunning := func() error {
		b, err := master.SSH("./kubectl get pod nginx -o=template -t={{.status.phase}}")
		if err != nil {
			return err
		}
		if !bytes.Equal(b, []byte("Running")) {
			return fmt.Errorf("nginx pod not running: %s", b)
		}
		return nil
	}
	if err := util.Retry(10, 5*time.Second, podIsRunning); err != nil {
		return err
	}

	// delete pod
	_, err := master.SSH("./kubectl delete pods nginx")
	if err != nil {
		return err
	}

	return nil
}
Example #16
0
func nodeCheck(master platform.Machine, nodes []platform.Machine) error {
	b, err := master.SSH("./kubectl get nodes")
	if err != nil {
		return err
	}

	// parse kubectl output for IPs
	addrMap := map[string]struct{}{}
	for _, line := range strings.Split(string(b), "\n")[1:] {
		addrMap[strings.SplitN(line, " ", 2)[0]] = struct{}{}
	}

	if len(addrMap) != len(nodes) {
		return fmt.Errorf("cannot detect all nodes in kubectl output \n%v\n%v", addrMap, nodes)
	}
	for _, node := range nodes {
		if _, ok := addrMap[node.PrivateIP()]; !ok {
			return fmt.Errorf("node IP missing from kubectl get nodes")
		}
	}
	return nil
}
Example #17
0
func doStart(m platform.Machine, version int, block bool) error {
	// start etcd instance
	var etcdStart string
	if version == 1 {
		etcdStart = "sudo systemctl start etcd.service"
	} else if version == 2 {
		etcdStart = "sudo systemctl start etcd2.service"
	} else {
		return fmt.Errorf("etcd version unspecified")
	}

	if !block {
		etcdStart += " --no-block"
	}

	_, err := m.SSH(etcdStart)
	if err != nil {
		return fmt.Errorf("SSH cmd to %v failed: %s", m.IP(), err)
	}

	return nil
}
Example #18
0
func copyFile(mach platform.Machine, r io.Reader, path string) error {
	sess, err := mach.SSHSession()
	if err != nil {
		return err
	}
	defer sess.Close()

	wr, err := sess.StdinPipe()
	if err != nil {
		return err
	}

	if err := sess.Start(fmt.Sprintf("tee %s", path)); err != nil {
		return err
	}

	go func() {
		io.Copy(wr, r)
		wr.Close()
	}()

	return sess.Wait()
}
Example #19
0
// https://coreos.com/kubernetes/docs/latest/configure-kubectl.html
func configureKubectl(m platform.Machine, server string, version string) error {
	// ignore suffix like '-coreos.1' on version to grab upstream
	semverPrefix := regexp.MustCompile(`^v[\d]+\.[\d]+\.[\d]+`)
	version = semverPrefix.FindString(version)
	if version == "" {
		return fmt.Errorf("Failure to parse kubectl version")
	}

	var (
		ca        = "/home/core/ca.pem"
		adminKey  = "/home/core/admin-key.pem"
		adminCert = "/home/core/admin.pem"
		kubeURL   = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%v/bin/linux/amd64/kubectl", version)
	)

	if _, err := m.SSH("wget -q " + kubeURL); err != nil {
		return err
	}
	if _, err := m.SSH("chmod +x ./kubectl"); err != nil {
		return err
	}

	// cmds to configure kubectl
	cmds := []string{
		fmt.Sprintf("./kubectl config set-cluster default-cluster --server=https://%v --certificate-authority=%v", server, ca),
		fmt.Sprintf("./kubectl config set-credentials default-admin --certificate-authority=%v --client-key=%v --client-certificate=%v", ca, adminKey, adminCert),
		"./kubectl config set-context default-system --cluster=default-cluster --user=default-admin",
		"./kubectl config use-context default-system",
	}
	for _, cmd := range cmds {
		b, err := m.SSH(cmd)
		if err != nil {
			return fmt.Errorf("Failed on cmd: %s with error: %s and output %s", cmd, err, b)
		}
	}
	return nil
}