예제 #1
0
파일: nspawn.go 프로젝트: jonboulle/fleet
func (nc *nspawnCluster) createMember(id string) (m Member, err error) {
	nm := nspawnMember{
		uuid: util.NewMachineID(),
		id:   id,
		ip:   fmt.Sprintf("172.18.1.%s", id),
	}
	nc.members[nm.ID()] = nm

	basedir := path.Join(os.TempDir(), nc.name)
	fsdir := path.Join(basedir, nm.ID(), "fs")
	cmds := []string{
		// set up directory for fleet service
		fmt.Sprintf("mkdir -p %s/etc/systemd/system", fsdir),

		// minimum requirements for running systemd/coreos in a container.
		// NOTE: copying /etc/pam.d is necessary only for such setups with
		// sys-auth/pambase installed, for example developer image of CoreOS
		// 1053.2.0. It should be fine also for systems where /etc/pam.d is
		// empty, because then it should automatically fall back to
		// /usr/lib64/pam.d, which belongs to sys-libs/pam.
		fmt.Sprintf("mkdir -p %s/usr", fsdir),
		fmt.Sprintf("cp /etc/os-release %s/etc", fsdir),
		fmt.Sprintf("cp -a /etc/pam.d %s/etc", fsdir),
		fmt.Sprintf("ln -s /proc/self/mounts %s/etc/mtab", fsdir),
		fmt.Sprintf("ln -s usr/lib64 %s/lib64", fsdir),
		fmt.Sprintf("ln -s lib64 %s/lib", fsdir),
		fmt.Sprintf("ln -s usr/bin %s/bin", fsdir),
		fmt.Sprintf("ln -s usr/sbin %s/sbin", fsdir),
		fmt.Sprintf("mkdir -p %s/home/core/.ssh", fsdir),
		fmt.Sprintf("install -d -o root -g systemd-journal -m 2755 %s/var/log/journal", fsdir),
		fmt.Sprintf("chown -R 500:500 %s/home/core", fsdir),

		// We don't need this, and it's slow, so mask it
		fmt.Sprintf("ln -s /dev/null %s/etc/systemd/system/systemd-udev-hwdb-update.service", fsdir),

		// set up directory for sshd_config (see below)
		fmt.Sprintf("mkdir -p %s/etc/ssh", fsdir),
	}

	for _, cmd := range cmds {
		var stderr, stdout string
		stdout, stderr, err = run(cmd)
		if err != nil {
			log.Printf("Command '%s' failed:\nstdout:: %s\nstderr: %s\nerr: %v", cmd, stdout, stderr, err)
			return
		}
	}

	filesContents := []struct {
		path     string
		contents string
		mode     os.FileMode
	}{
		{
			"/etc/ssh/sshd_config",
			`# Use most defaults for sshd configuration.
			UsePrivilegeSeparation sandbox
			Subsystem sftp internal-sftp
			UseDNS no
			`,
			0644,
		},
		// For expediency, generate the minimal viable SSH keys for the host, instead of the default set
		{
			"/etc/systemd/system/sshd-keygen.service",
			`[Unit]
			Description=Generate sshd host keys
			Before=sshd.service

			[Service]
			Type=oneshot
			RemainAfterExit=yes
			ExecStart=/usr/bin/ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N "" -b 1024`,
			0644,
		},
		{
			"/etc/passwd",
			"core:x:500:500:CoreOS Admin:/home/core:/bin/bash",
			0644,
		},
		{
			"/etc/group",
			"core:x:500:",
			0644,
		},
		{
			"/etc/machine-id",
			nm.ID(),
			0644,
		},
		{
			"/home/core/.bash_profile",
			"export PATH=/opt/fleet:$PATH",
			0644,
		},
	}

	for _, file := range filesContents {
		if err = ioutil.WriteFile(path.Join(fsdir, file.path), []byte(file.contents), file.mode); err != nil {
			log.Printf("Failed writing %s: %v", path.Join(fsdir, file.path), err)
			return
		}
	}

	if err = nc.insertBin(fleetdBinPath, fsdir); err != nil {
		log.Printf("Failed preparing fleetd in filesystem: %v", err)
		return
	}

	if err = nc.insertBin(fleetctlBinPath, fsdir); err != nil {
		log.Printf("Failed preparing fleetctl in filesystem: %v", err)
		return
	}

	if err = nc.buildConfigDrive(fsdir, nm.IP()); err != nil {
		log.Printf("Failed building config drive: %v", err)
		return
	}

	exec := strings.Join([]string{
		"/usr/bin/systemd-nspawn",
		"--bind-ro=/usr",
		"-b",
		"--uuid=" + nm.uuid,
		fmt.Sprintf("-M %s%s", nc.name, nm.ID()),
		"--capability=CAP_NET_BIND_SERVICE,CAP_SYS_TIME", // needed for ntpd
		"--network-bridge fleet0",
		fmt.Sprintf("-D %s", fsdir),
	}, " ")
	log.Printf("Creating nspawn container: %s", exec)
	err = nc.systemd(fmt.Sprintf("%s%s.service", nc.name, nm.ID()), exec)
	if err != nil {
		log.Printf("Failed creating nspawn container: %v", err)
		return
	}

	nm.pid, err = nc.machinePID(nm.ID())
	if err != nil {
		log.Printf("Failed detecting machine %s%s PID: %v", nc.name, nm.ID(), err)
		return
	}

	alarm := time.After(10 * time.Second)
	addr := fmt.Sprintf("%s:%d", nm.IP(), fleetAPIPort)
	for {
		select {
		case <-alarm:
			err = fmt.Errorf("Timed out waiting for machine to start")
			log.Printf("Starting %s%s failed: %v", nc.name, nm.ID(), err)
			return
		default:
		}
		log.Printf("Dialing machine: %s", addr)
		c, err := net.DialTimeout("tcp", addr, 100*time.Millisecond)
		if err == nil {
			c.Close()
			break
		}
		time.Sleep(100 * time.Millisecond)
	}

	return Member(&nm), nil
}
예제 #2
0
파일: node_test.go 프로젝트: pulcy/j2
// TestDetectMachineId checks for etcd registration failing on a duplicated
// machine-id on different machines.
// First it creates a cluster with 2 members, m0 and m1. Then make their
// machine IDs the same as each other, by explicitly setting the m1's ID to
// the same as m0's. Test succeeds when an error returns, while test fails
// when nothing happens.
func TestDetectMachineId(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy(t)

	members, err := platform.CreateNClusterMembers(cluster, 2)
	if err != nil {
		t.Fatal(err)
	}

	m0 := members[0]
	m1 := members[1]
	_, err = cluster.WaitForNMachines(m0, 2)
	if err != nil {
		t.Fatal(err)
	}

	machineIdFile := "/etc/machine-id"

	// Restart fleet service, and check if its systemd status is still active.
	restartFleetService := func(m platform.Member) error {
		stdout, err := cluster.MemberCommand(m, "sudo", "systemctl", "restart", "fleet.service")
		if err != nil {
			return fmt.Errorf("Failed to restart fleet service\nstdout: %s\nerr: %v", stdout, err)
		}

		stdout, _ = cluster.MemberCommand(m, "systemctl", "show", "--property=ActiveState", "fleet")
		if strings.TrimSpace(stdout) != "ActiveState=active" {
			return fmt.Errorf("Fleet unit not reported as active: %s", stdout)
		}
		stdout, _ = cluster.MemberCommand(m, "systemctl", "show", "--property=Result", "fleet")
		if strings.TrimSpace(stdout) != "Result=success" {
			return fmt.Errorf("Result for fleet unit not reported as success: %s", stdout)
		}
		return nil
	}

	stdout, err := cluster.MemberCommand(m0, "cat", machineIdFile)
	if err != nil {
		t.Fatalf("Failed to get machine-id\nstdout: %s\nerr: %v", stdout, err)
	}
	m0_machine_id := strings.TrimSpace(stdout)

	// If the two machine IDs are different with each other,
	// set the m1's ID to the same one as m0, to intentionally
	// trigger an error case of duplication of machine ID.
	stdout, err = cluster.MemberCommand(m1,
		"echo", m0_machine_id, "|", "sudo", "tee", machineIdFile)
	if err != nil {
		t.Fatalf("Failed to replace machine-id\nstdout: %s\nerr: %v", stdout, err)
	}

	if err := restartFleetService(m1); err != nil {
		t.Fatal(err)
	}

	// fleetd should actually be running, but failing to list machines.
	// So we should expect a specific error after running fleetctl list-machines,
	// like "googlapi: Error 503: fleet server unable to communicate with etcd".
	stdout, stderr, err := cluster.Fleetctl(m1, "list-machines", "--no-legend")
	if err != nil {
		if !strings.Contains(err.Error(), "exit status 1") ||
			!strings.Contains(stderr, "fleet server unable to communicate with etcd") {
			t.Fatalf("m1: Failed to get list of machines. err: %v\nstderr: %s", err, stderr)
		}
		// If both conditions are satisfied, "exit status 1" and
		// "...unable to communicate...", then it's an expected error. PASS.
	} else {
		t.Fatalf("m1: should get an error, but got success.\nstderr: %s", stderr)
	}

	// Trigger another test case of m0's ID getting different from m1's.
	// Then it's expected that m0 and m1 would be working properly with distinct
	// machine IDs, after having restarted fleet.service both on m0 and m1.
	stdout, err = cluster.MemberCommand(m0,
		"echo", util.NewMachineID(), "|", "sudo", "tee", machineIdFile)
	if err != nil {
		t.Fatalf("m0: Failed to replace machine-id\nstdout: %s\nerr: %v", stdout, err)
	}

	// Restart fleet service on m0, and see that it's still working.
	if err := restartFleetService(m0); err != nil {
		t.Fatal(err)
	}

	stdout, stderr, err = cluster.Fleetctl(m0, "list-machines", "--no-legend")
	if err != nil {
		t.Fatalf("m0: error: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
	}
}