func restartDockerHost(node remotessh.TestbedNode) error {
logrus.Infof("Restarting docker on %q", node.GetName())
// note that for all these restart tasks we error out quietly to avoid other
// hosts being cleaned up
node.RunCommand("sudo service docker restart")
return nil
}
func restartNetplugin(node remotessh.TestbedNode) error {
logrus.Infof("Restarting netplugin on %q", node.GetName())
err := node.RunCommand("sudo systemctl restart netplugin netmaster")
if err != nil {
return err
}
time.Sleep(5 * time.Second)
return nil
}
func waitForVolplugin(node remotessh.TestbedNode) error {
logrus.Infof("Checking if volplugin is running on %q", node.GetName())
err := runCommandUntilNoError(node, "docker inspect -f {{.State.Running}} volplugin | grep true", 30)
if err == nil {
logrus.Infof("Volplugin is running on %q", node.GetName())
}
return nil
}
func (s *systemtestSuite) clearContainerHost(node remotessh.TestbedNode) error {
startedContainers.Lock()
names := []string{}
for name := range startedContainers.names {
names = append(names, name)
}
startedContainers.Unlock()
logrus.Infof("Clearing containers %v on %q", names, node.GetName())
node.RunCommand(fmt.Sprintf("docker rm -f %s", strings.Join(names, " ")))
return nil
}
func runCommandUntilNoError(node remotessh.TestbedNode, cmd string, timeout int) error {
runCmd := func() (string, bool) {
if err := node.RunCommand(cmd); err != nil {
return "", false
}
return "", true
}
timeoutMessage := fmt.Sprintf("timeout reached trying to run %v on %q", cmd, node.GetName())
_, err := WaitForDone(runCmd, 10*time.Millisecond, time.Duration(timeout)*time.Second, timeoutMessage)
return err
}
func waitDockerizedServicesHost(node remotessh.TestbedNode) error {
services := map[string]string{
"etcd": "etcdctl cluster-health",
}
for s, cmd := range services {
logrus.Infof("Waiting for %s on %q", s, node.GetName())
out, err := WaitForDone(
func() (string, bool) {
out, err := node.RunCommandWithOutput(cmd)
if err != nil {
return out, false
}
return out, true
}, 2*time.Second, time.Minute, fmt.Sprintf("service %s is not healthy", s))
if err != nil {
logrus.Infof("a dockerized service failed. Output: %s, Error: %v", out, err)
return err
}
}
return nil
}
func waitForAPIServer(node remotessh.TestbedNode) error {
logrus.Infof("Checking if apiserver is running on %q", node.GetName())
err := runCommandUntilNoError(node, "docker inspect -f {{.State.Running}} apiserver | grep true", 30)
if err == nil {
logrus.Infof("APIServer is running on %q", node.GetName())
}
then := time.Now()
err = runCommandUntilNoError(node, "connwait 127.0.0.1:9005", 60)
if err != nil {
return err
}
logrus.Infof("Took %s for apiserver on %q to be accessible", time.Since(then), node.GetName())
return nil
}
func (s *systemtestSuite) clearVolumeHost(node remotessh.TestbedNode) error {
logrus.Infof("Clearing volumes on %q", node.GetName())
node.RunCommand("docker volume ls | tail -n +2 | awk '{ print $2 }' | xargs docker volume rm")
return nil
}
func stopVolplugin(node remotessh.TestbedNode) error {
logrus.Infof("Stopping the volplugin on %q", node.GetName())
defer time.Sleep(time.Second)
return node.RunCommand("sudo systemctl stop volplugin")
}
func startVolplugin(node remotessh.TestbedNode) error {
logrus.Infof("Starting the volplugin on %q", node.GetName())
return node.RunCommandBackground("sudo systemctl start volplugin")
}
func stopAPIServer(node remotessh.TestbedNode) error {
logrus.Infof("Stopping the apiserver on %q", node.GetName())
defer time.Sleep(time.Second)
return node.RunCommand("sudo systemctl stop apiserver")
}
func startAPIServer(node remotessh.TestbedNode) error {
logrus.Infof("Starting the apiserver on %q", node.GetName())
err := node.RunCommandBackground("sudo systemctl start apiserver")
logrus.Infof("Waiting for apiserver startup on %q", node.GetName())
return err
}