// TestScheduleReplace starts 1 unit, followed by starting another unit
// that replaces the 1st unit. Then it verifies that the 2 units are
// started on different machines.
func TestScheduleReplace(t *testing.T) {
cluster, err := platform.NewNspawnCluster("smoke")
if err != nil {
t.Fatal(err)
}
defer cluster.Destroy(t)
// Start with a simple three-node cluster
members, err := platform.CreateNClusterMembers(cluster, 2)
if err != nil {
t.Fatal(err)
}
m0 := members[0]
if _, err := cluster.WaitForNMachines(m0, 2); err != nil {
t.Fatal(err)
}
// Start a unit without Replaces
uNames := []string{
"fixtures/units/replace.0.service",
"fixtures/units/replace.1.service",
}
if stdout, stderr, err := cluster.Fleetctl(m0, "start", "--no-block", uNames[0]); err != nil {
t.Fatalf("Failed starting unit %s: \nstdout: %s\nstderr: %s\nerr: %v", uNames[0], stdout, stderr, err)
}
active, err := cluster.WaitForNActiveUnits(m0, 1)
if err != nil {
t.Fatal(err)
}
_, err = util.ActiveToSingleStates(active)
if err != nil {
t.Fatal(err)
}
// Start a unit that replaces the former one, replace.0.service
if stdout, stderr, err := cluster.Fleetctl(m0, "start", "--no-block", uNames[1]); err != nil {
t.Fatalf("Failed starting unit %s: \nstdout: %s\nstderr: %s\nerr: %v", uNames[1], stdout, stderr, err)
}
// Check that both units should show up
stdout, _, err := cluster.Fleetctl(m0, "list-unit-files", "--no-legend")
if err != nil {
t.Fatalf("Failed to run list-unit-files: %v", err)
}
units := strings.Split(strings.TrimSpace(stdout), "\n")
if len(units) != 2 {
t.Fatalf("Did not find two units in cluster: \n%s", stdout)
}
active, err = cluster.WaitForNActiveUnits(m0, 2)
if err != nil {
t.Fatal(err)
}
states, err := util.ActiveToSingleStates(active)
if err != nil {
t.Fatal(err)
}
// Check that the unit 1 is located on a different machine from that of unit 0
nUnits := 2
uNameBase := make([]string, nUnits)
machs := make([]string, nUnits)
for i, uName := range uNames {
uNameBase[i] = path.Base(uName)
machs[i] = states[uNameBase[i]].Machine
}
if machs[0] == machs[1] {
t.Fatalf("machine for %s is %s, the same as that of %s.", uNameBase[0], machs[0], uNameBase[1])
}
// Check that circular replaces end up with 1 launched unit.
// First of all, stop the existing unit replace.0.service.
if stdout, stderr, err := cluster.Fleetctl(m0, "destroy", uNameBase[0]); err != nil {
t.Fatalf("Failed to destroy unit %s: \nstdout: %s\nstderr: %s\nerr: %v",
uNameBase[0], stdout, stderr, err)
}
// Generate a new service 0 derived by a fixture, make the new service
// replace service 1, and store it under /tmp.
uName0tmp := path.Join("/tmp", uNameBase[0])
err = util.GenNewFleetService(uName0tmp, uNames[1],
"Replaces=replace.1.service", "Replaces=replace.0.service")
if err != nil {
t.Fatalf("Failed to generate a temp fleet service: %v", err)
}
// Start replace.0 unit that replaces replace.1.service,
// then fleetctl list-unit-files should show only return 1 launched unit.
// Note that we still need to run list-units once, before doing
// list-unit-files, for reliable tests.
stdout, stderr, err := cluster.Fleetctl(m0, "start", "--no-block", uName0tmp)
if err != nil {
t.Fatalf("Failed starting unit %s: \nstdout: %s\nstderr: %s\nerr: %v",
uName0tmp, stdout, stderr, err)
}
stdout, _, err = cluster.Fleetctl(m0, "list-unit-files", "--no-legend")
if err != nil {
t.Fatalf("Failed to run list-unit-files: %v", err)
}
units = strings.Split(strings.TrimSpace(stdout), "\n")
if len(units) != nUnits {
t.Fatalf("Did not find two units in cluster: \n%s", stdout)
}
_, err = cluster.WaitForNActiveUnits(m0, nUnits)
if err != nil {
t.Fatal(err)
}
ufs, err := cluster.WaitForNUnitFiles(m0, nUnits)
if err != nil {
t.Fatalf("Failed to run list-unit-files: %v", err)
}
uStates := make([][]util.UnitFileState, nUnits)
var found bool
for i, unb := range uNameBase {
uStates[i], found = ufs[unb]
if len(ufs) != nUnits || !found {
t.Fatalf("Did not find %d launched unit as expected: got %d\n", nUnits, len(ufs))
}
}
nLaunched := 0
for _, us := range uStates {
for _, state := range us {
if strings.Contains(state.State, "launched") {
nLaunched += 1
}
}
}
if nLaunched != 1 {
t.Fatalf("Did not find 1 launched unit as expected: got %d", nLaunched)
}
os.Remove(uName0tmp)
}
// replaceUnitCommon() tests whether a command "fleetctl {submit,load,start}
// --replace hello.service" works or not.
func replaceUnitCommon(t *testing.T, cmd string, numRUnits int) error {
// check if cmd is one of the supported commands.
listCmds := []string{"submit", "load", "start"}
found := false
for _, ccmd := range listCmds {
if ccmd == cmd {
found = true
}
}
if !found {
return fmt.Errorf("invalid command %s", cmd)
}
cluster, err := platform.NewNspawnCluster("smoke")
if err != nil {
return fmt.Errorf("%v", err)
}
defer cluster.Destroy(t)
m, err := cluster.CreateMember()
if err != nil {
return fmt.Errorf("%v", err)
}
_, err = cluster.WaitForNMachines(m, 1)
if err != nil {
return fmt.Errorf("%v", err)
}
if _, err := os.Stat(tmpFixtures); os.IsNotExist(err) {
os.Mkdir(tmpFixtures, 0755)
}
prepareReplaceUnits := func(cmd string, unitFiles []string, numUnits int) (bodiesOrig []string, err error) {
for i, helloFilename := range unitFiles {
tmpHelloFixture := fmt.Sprintf("/tmp/fixtures/hello@%d.service", i)
err = util.CopyFile(tmpHelloFixture, fxtHelloService)
if err != nil {
return nil, fmt.Errorf("Failed to copy a temp fleet service: %v", err)
}
// retrieve content of hello.service, and append to bodiesOrig[]
bodyCur, _, err := cluster.Fleetctl(m, "cat", helloFilename)
if err != nil {
return nil, fmt.Errorf("Failed to run cat %s: %v", helloFilename, err)
}
bodiesOrig = append(bodiesOrig, bodyCur)
// generate a new service derived by fixtures, and store it under /tmp
curHelloService := path.Join("/tmp", helloFilename)
err = util.GenNewFleetService(curHelloService, fxtHelloService, "sleep 2", "sleep 1")
if err != nil {
return nil, fmt.Errorf("Failed to generate a temp fleet service: %v", err)
}
}
return bodiesOrig, nil
}
compareReplaceUnits := func(cmd string, unitFiles []string, bodiesOrig []string, numUnits int) (err error) {
for i, helloFilename := range unitFiles {
curHelloService := path.Join("/tmp", helloFilename)
// replace the unit and assert it shows up
if _, _, err = cluster.Fleetctl(m, cmd, "--replace", curHelloService); err != nil {
return fmt.Errorf("Unable to replace fleet unit: %v", err)
}
if err := waitForNUnitsCmd(cluster, m, cmd, numUnits); err != nil {
return fmt.Errorf("Did not find %d units in cluster", numUnits)
}
// retrieve content of hello.service, and compare it with the
// correspondent entry in bodiesOrig[]
bodyCur, _, err := cluster.Fleetctl(m, "cat", helloFilename)
if err != nil {
return fmt.Errorf("Failed to run cat %s: %v", helloFilename, err)
}
if bodiesOrig[i] == bodyCur {
return fmt.Errorf("Error. the unit %s has not been replaced.", helloFilename)
}
}
return nil
}
// Launch units for the initial setup, and make sure that all units
// are actually available via fleectl list-{units,unit-files}.
unitFiles, err := launchUnitsCmd(cluster, m, cmd, numRUnits)
if err != nil {
return err
}
if err := waitForNUnitsCmd(cluster, m, cmd, numRUnits); err != nil {
return fmt.Errorf("Did not find %d units in cluster", numRUnits)
}
// Before starting comparison, prepare a slice of unit bodies of each
// unit file.
bodiesOrig, err := prepareReplaceUnits(cmd, unitFiles, numRUnits)
if err != nil {
return err
}
// Replace each unit with a new one, and compare its body with the original
// unit body, to make sure that "fleetctl <cmd> --replace" actually worked.
if err := compareReplaceUnits(cmd, unitFiles, bodiesOrig, numRUnits); err != nil {
return err
}
// clean up units via corresponding destroy commands,
// also remove temp files under /tmp.
if err := cleanUnits(cluster, m, cleanCmd[cmd], unitFiles, numRUnits); err != nil {
return err
}
for i := 1; i <= numRUnits; i++ {
os.Remove(fmt.Sprintf("/tmp/hello@%d.service", i))
}
if err := waitForNUnitsCmd(cluster, m, cmd, 0); err != nil {
return fmt.Errorf("Failed to get every unit to be cleaned up: %v", err)
}
os.Remove(tmpFixtures)
return nil
}
// TestReplaceSerialization tests if the ExecStartPre of the new version
// of the unit when it replaces the old one is excuted after
// ExecStopPost of the old version.
// This test is to make sure that two versions of the same unit will not
// conflict with each other, that the directives are always serialized,
// and it tries its best to avoid the following scenarios:
// https://github.com/coreos/fleet/issues/1000
// https://github.com/systemd/systemd/issues/518
// Now we can't guarantee that that behaviour will not be triggered by
// another external operation, but at least from the Unit replace
// feature context we try to avoid it.
func TestReplaceSerialization(t *testing.T) {
cluster, err := platform.NewNspawnCluster("smoke")
if err != nil {
t.Fatal(err)
}
defer cluster.Destroy(t)
m, err := cluster.CreateMember()
if err != nil {
t.Fatal(err)
}
_, err = cluster.WaitForNMachines(m, 1)
if err != nil {
t.Fatal(err)
}
tmpSyncFile := "/tmp/fleetSyncReplaceFile"
syncOld := "echo 'sync'"
syncNew := fmt.Sprintf("test -f %s", tmpSyncFile)
tmpSyncService := "/tmp/replace-sync.service"
syncService := "fixtures/units/replace-sync.service"
stdout, stderr, err := cluster.Fleetctl(m, "start", syncService)
if err != nil {
t.Fatalf("Unable to start unit: \nstdout: %s\nstderr: %s\nerr: %v", stdout, stderr, err)
}
_, err = cluster.WaitForNActiveUnits(m, 1)
if err != nil {
t.Fatal(err)
}
// replace the unit content, make sure that:
// It shows up and it did 'test -f /tmp/fleetSyncReplaceFile' correctly
err = util.GenNewFleetService(tmpSyncService, syncService, syncNew, syncOld)
if err != nil {
t.Fatalf("Failed to generate a temp fleet service: %v", err)
}
stdout, stderr, err = cluster.Fleetctl(m, "start", "--replace", tmpSyncService)
if err != nil {
t.Fatalf("Failed to replace unit: \nstdout: %s\nstderr: %s\nerr: %v", stdout, stderr, err)
}
_, err = cluster.WaitForNActiveUnits(m, 1)
if err != nil {
t.Fatalf("Did not find 1 unit in cluster, unit replace failed: %v", err)
}
// Wait for the sync file, if the sync file is not created then
// the previous unit failed, if it's created we continue. Here
// the new version of the unit is probably already running and
// the ExecStartPre is running at the same time, if it failed
// then we probably will catch it later when we check its status
tmpService := path.Base(tmpSyncService)
timeout, err := util.WaitForState(
func() bool {
_, err = cluster.MemberCommand(m, syncNew)
if err != nil {
return false
}
return true
},
)
if err != nil {
t.Fatalf("Failed to check if file %s exists within %v", tmpSyncFile, timeout)
}
timeout, err = util.WaitForState(
func() bool {
stdout, _ = cluster.MemberCommand(m, "systemctl", "show", "--property=ActiveState", tmpService)
if strings.TrimSpace(stdout) != "ActiveState=active" {
return false
}
return true
},
)
if err != nil {
t.Fatalf("%s unit not reported as active within %v", tmpService, timeout)
}
timeout, err = util.WaitForState(
func() bool {
stdout, _ = cluster.MemberCommand(m, "systemctl", "show", "--property=Result", tmpService)
if strings.TrimSpace(stdout) != "Result=success" {
return false
}
return true
},
)
if err != nil {
t.Fatalf("Result for %s unit not reported as success withing %v", tmpService, timeout)
}
os.Remove(tmpSyncFile)
os.Remove(tmpSyncService)
}
// TestScheduleCircularReplace starts 2 units that tries to replace each other.
// Thus it's expected that only one of the units becomes active.
func TestScheduleCircularReplace(t *testing.T) {
cluster, err := platform.NewNspawnCluster("smoke")
if err != nil {
t.Fatal(err)
}
defer cluster.Destroy(t)
members, err := platform.CreateNClusterMembers(cluster, 2)
if err != nil {
t.Fatal(err)
}
m0 := members[0]
if _, err := cluster.WaitForNMachines(m0, 2); err != nil {
t.Fatal(err)
}
// Check that circular replaces end up with 1 launched unit.
// To do that, generate a new service 0 that replaces service 1, and store
// it under /tmp. Also store the original service 1 that replace 0.
uNames := []string{
"fixtures/units/replace.0.service",
"fixtures/units/replace.1.service",
}
nUnits := 2
nActiveUnits := 1
uNameBase := make([]string, nUnits)
for i, uName := range uNames {
uNameBase[i] = path.Base(uName)
}
uName0tmp := path.Join("/tmp", uNameBase[0])
err = util.GenNewFleetService(uName0tmp, uNames[1],
"Replaces=replace.1.service", "Replaces=replace.0.service")
if err != nil {
t.Fatalf("Failed to generate a temp fleet service: %v", err)
}
// Start replace.0 unit that replaces replace.1.service,
// then fleetctl list-unit-files should show only return 1 launched unit.
stdout, stderr, err := cluster.Fleetctl(m0, "start", "--no-block", uName0tmp)
if err != nil {
t.Fatalf("Failed starting unit %s: \nstdout: %s\nstderr: %s\nerr: %v",
uName0tmp, stdout, stderr, err)
}
stdout, stderr, err = cluster.Fleetctl(m0, "list-unit-files", "--no-legend")
if err != nil {
t.Fatalf("Failed to run list-unit-files:\nstdout: %s\nstderr: %s\nerr: %v", stdout, stderr, err)
}
units := strings.Split(strings.TrimSpace(stdout), "\n")
if len(units) != nActiveUnits {
t.Fatalf("Did not find two units in cluster: \n%s", stdout)
}
_, err = cluster.WaitForNActiveUnits(m0, nActiveUnits)
if err != nil {
t.Fatal(err)
}
ufs, err := cluster.WaitForNUnitFiles(m0, nActiveUnits)
if err != nil {
t.Fatalf("Failed to run list-unit-files: %v", err)
}
// Start replace.1 unit that replaces replace.0.service,
// and then check that only 1 unit is active
if stdout, stderr, err := cluster.Fleetctl(m0, "start", "--no-block", uNames[1]); err != nil {
t.Fatalf("Failed starting unit %s: \nstdout: %s\nstderr: %s\nerr: %v", uNames[1], stdout, stderr, err)
}
stdout, stderr, err = cluster.Fleetctl(m0, "list-unit-files", "--no-legend")
if err != nil {
t.Fatalf("Failed to run list-unit-files:\nstdout: %s\nstderr: %s\nerr: %v", stdout, stderr, err)
}
units = strings.Split(strings.TrimSpace(stdout), "\n")
if len(units) != nUnits {
t.Fatalf("Did not find %d units in cluster: \n%s", nUnits, stdout)
}
active, err := cluster.WaitForNActiveUnits(m0, nActiveUnits)
if err != nil {
t.Fatal(err)
}
_, err = util.ActiveToSingleStates(active)
if err != nil {
t.Fatal(err)
}
uStates := make([][]util.UnitFileState, nUnits)
for i, unb := range uNameBase {
uStates[i], _ = ufs[unb]
}
nLaunched := 0
for _, us := range uStates {
for _, state := range us {
if strings.Contains(state.State, "launched") {
nLaunched += 1
}
}
}
if nLaunched != nActiveUnits {
t.Fatalf("Did not find %d launched unit as expected: got %d", nActiveUnits, nLaunched)
}
os.Remove(uName0tmp)
}