func (s *ZZBackupSuite) testClusterBackup(t *c.C, index int, path string) {
debugf(t, "restoring cluster backup %s", filepath.Base(path))
// boot the cluster using an RFC 5737 TEST-NET IP, avoiding conflicts
// with those used by script/bootstrap-flynn so the test can be run in
// development
ip := fmt.Sprintf("192.0.2.%d", index+100)
device := fmt.Sprintf("eth0:%d", index+10)
t.Assert(run(t, exec.Command("sudo", "ifconfig", device, ip)), Succeeds)
dir := t.MkDir()
debugf(t, "using tempdir %s", dir)
debug(t, "starting flynn-host")
cmd := exec.Command(
"sudo",
"../host/bin/flynn-host",
"daemon",
"--id", fmt.Sprintf("backup%d", index),
"--external-ip", ip,
"--listen-ip", ip,
"--bridge-name", fmt.Sprintf("backupbr%d", index),
"--state", filepath.Join(dir, "host-state.bolt"),
"--volpath", filepath.Join(dir, "volumes"),
"--log-dir", filepath.Join(dir, "logs"),
"--flynn-init", "../host/bin/flynn-init",
)
out, err := os.Create(filepath.Join(dir, "flynn-host.log"))
t.Assert(err, c.IsNil)
defer out.Close()
cmd.Stdout = out
cmd.Stderr = out
t.Assert(cmd.Start(), c.IsNil)
go cmd.Process.Wait()
defer func() {
// collect-debug-info if the tests failed then kill flynn-host
if t.Failed() {
cmd := exec.Command(
"sudo",
"-E",
"../host/bin/flynn-host",
"collect-debug-info",
"--log-dir", filepath.Join(dir, "logs"),
)
cmd.Env = []string{fmt.Sprintf("DISCOVERD=%s:1111", ip)}
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Run()
}
exec.Command("sudo", "kill", strconv.Itoa(cmd.Process.Pid)).Run()
}()
debugf(t, "bootstrapping flynn from backup")
cmd = exec.Command(
"../host/bin/flynn-host",
"bootstrap",
"--peer-ips", ip,
"--from-backup", path,
"../bootstrap/bin/manifest.json",
)
cmd.Env = []string{
"CLUSTER_DOMAIN=1.localflynn.com",
fmt.Sprintf("DISCOVERD=%s:1111", ip),
fmt.Sprintf("FLANNEL_NETWORK=100.%d.0.0/16", index+101),
}
logR, logW := io.Pipe()
defer logW.Close()
go func() {
buf := bufio.NewReader(logR)
for {
line, err := buf.ReadString('\n')
if err != nil {
return
}
debug(t, line[0:len(line)-1])
}
}()
cmd.Stdout = logW
cmd.Stderr = logW
t.Assert(cmd.Run(), c.IsNil)
debug(t, "waiting for nodejs-web service")
disc := discoverd.NewClientWithURL(fmt.Sprintf("http://%s:1111", ip))
_, err = disc.Instances("nodejs-web", 30*time.Second)
t.Assert(err, c.IsNil)
debug(t, "checking HTTP requests")
req, err := http.NewRequest("GET", "http://"+ip, nil)
t.Assert(err, c.IsNil)
req.Host = "nodejs.1.localflynn.com"
var res *http.Response
// try multiple times in case we get a 503 from the router as it has
// not seen the service yet
err = attempt.Strategy{Total: 10 * time.Second, Delay: 100 * time.Millisecond}.Run(func() (err error) {
res, err = http.DefaultClient.Do(req)
if err != nil {
return err
} else if res.StatusCode == http.StatusServiceUnavailable {
return errors.New("router returned 503")
}
return nil
})
t.Assert(err, c.IsNil)
t.Assert(res.StatusCode, c.Equals, http.StatusOK)
debug(t, "getting app release")
controllerInstances, err := disc.Instances("controller", 30*time.Second)
t.Assert(err, c.IsNil)
controllerURL := "http://" + controllerInstances[0].Addr
controllerKey := controllerInstances[0].Meta["AUTH_KEY"]
client, err := controller.NewClient(controllerURL, controllerKey)
t.Assert(err, c.IsNil)
release, err := client.GetAppRelease("nodejs")
t.Assert(err, c.IsNil)
debug(t, "configuring flynn CLI")
flynnrc := filepath.Join(dir, ".flynnrc")
conf := &config.Config{}
t.Assert(conf.Add(&config.Cluster{
Name: "default",
ControllerURL: controllerURL,
Key: controllerKey,
}, true), c.IsNil)
t.Assert(conf.SaveTo(flynnrc), c.IsNil)
flynn := func(cmdArgs ...string) *CmdResult {
cmd := exec.Command(args.CLI, cmdArgs...)
cmd.Env = flynnEnv(flynnrc)
cmd.Env = append(cmd.Env, "FLYNN_APP=nodejs")
return run(t, cmd)
}
if _, ok := release.Env["FLYNN_REDIS"]; ok {
debug(t, "checking redis resource")
// try multiple times as the Redis resource is not guaranteed to be up yet
var redisResult *CmdResult
err = attempt.Strategy{Total: 10 * time.Second, Delay: 100 * time.Millisecond}.Run(func() error {
redisResult = flynn("redis", "redis-cli", "--", "PING")
return redisResult.Err
})
t.Assert(err, c.IsNil)
t.Assert(redisResult, SuccessfulOutputContains, "PONG")
}
debug(t, "checking mysql resource")
if _, ok := release.Env["FLYNN_MYSQL"]; ok {
t.Assert(flynn("mysql", "console", "--", "-e", "SELECT * FROM foos"), SuccessfulOutputContains, "foobar")
} else {
t.Assert(flynn("resource", "add", "mysql"), Succeeds)
}
debug(t, "checking mongodb resource")
if _, ok := release.Env["FLYNN_MONGO"]; ok {
t.Assert(flynn("mongodb", "mongo", "--", "--eval", "db.foos.find()"), SuccessfulOutputContains, "foobar")
} else {
t.Assert(flynn("resource", "add", "mongodb"), Succeeds)
}
debug(t, "checking dashboard STATUS_KEY matches status AUTH_KEY")
dashboardStatusKeyResult := flynn("-a", "dashboard", "env", "get", "STATUS_KEY")
t.Assert(dashboardStatusKeyResult, Succeeds)
statusAuthKeyResult := flynn("-a", "status", "env", "get", "AUTH_KEY")
t.Assert(statusAuthKeyResult, Succeeds)
t.Assert(dashboardStatusKeyResult.Output, c.Equals, statusAuthKeyResult.Output)
}