func restartDockerHost(node remotessh.TestbedNode) error {
logrus.Infof("Restarting docker on %q", node.GetName())
// note that for all these restart tasks we error out quietly to avoid other
// hosts being cleaned up
node.RunCommand("sudo service docker restart")
return nil
}
func restartNetplugin(node remotessh.TestbedNode) error {
logrus.Infof("Restarting netplugin on %q", node.GetName())
err := node.RunCommand("sudo systemctl restart netplugin netmaster")
if err != nil {
return err
}
time.Sleep(5 * time.Second)
return nil
}
func waitForVolplugin(node remotessh.TestbedNode) error {
logrus.Infof("Checking if volplugin is running on %q", node.GetName())
err := runCommandUntilNoError(node, "docker inspect -f {{.State.Running}} volplugin | grep true", 30)
if err == nil {
logrus.Infof("Volplugin is running on %q", node.GetName())
}
return nil
}
func (s *systemtestSuite) clearContainerHost(node remotessh.TestbedNode) error {
startedContainers.Lock()
names := []string{}
for name := range startedContainers.names {
names = append(names, name)
}
startedContainers.Unlock()
logrus.Infof("Clearing containers %v on %q", names, node.GetName())
node.RunCommand(fmt.Sprintf("docker rm -f %s", strings.Join(names, " ")))
return nil
}
func runCommandUntilNoError(node remotessh.TestbedNode, cmd string, timeout int) error {
runCmd := func() (string, bool) {
if err := node.RunCommand(cmd); err != nil {
return "", false
}
return "", true
}
timeoutMessage := fmt.Sprintf("timeout reached trying to run %v on %q", cmd, node.GetName())
_, err := WaitForDone(runCmd, 10*time.Millisecond, time.Duration(timeout)*time.Second, timeoutMessage)
return err
}
func (s *systemtestSuite) CheckBgpNoConnectionForaNode(c *C, node remotessh.TestbedNode) error {
for i := 0; i < 100; i++ {
time.Sleep(3 * time.Second)
out, _ := node.RunCommandWithOutput("/opt/gopath/bin/gobgp neighbor")
fmt.Println(out)
if !strings.Contains(out, "Establ") {
return nil
}
}
return errors.New("BGP connection persists")
}
func waitDockerizedServicesHost(node remotessh.TestbedNode) error {
services := map[string]string{
"etcd": "etcdctl cluster-health",
}
for s, cmd := range services {
logrus.Infof("Waiting for %s on %q", s, node.GetName())
out, err := WaitForDone(
func() (string, bool) {
out, err := node.RunCommandWithOutput(cmd)
if err != nil {
return out, false
}
return out, true
}, 2*time.Second, time.Minute, fmt.Sprintf("service %s is not healthy", s))
if err != nil {
logrus.Infof("a dockerized service failed. Output: %s, Error: %v", out, err)
return err
}
}
return nil
}
func waitForAPIServer(node remotessh.TestbedNode) error {
logrus.Infof("Checking if apiserver is running on %q", node.GetName())
err := runCommandUntilNoError(node, "docker inspect -f {{.State.Running}} apiserver | grep true", 30)
if err == nil {
logrus.Infof("APIServer is running on %q", node.GetName())
}
then := time.Now()
err = runCommandUntilNoError(node, "connwait 127.0.0.1:9005", 60)
if err != nil {
return err
}
logrus.Infof("Took %s for apiserver on %q to be accessible", time.Since(then), node.GetName())
return nil
}
//ServiceLogs queries and returns upto maxLogLines lines from systemd service unit logs
func ServiceLogs(n remotessh.TestbedNode, srv string, maxLogLines int) (string, error) {
return n.RunCommandWithOutput(fmt.Sprintf("sudo systemctl status -ln%d %s", maxLogLines, srv))
}
//ServiceRestart restarts a systemd service unit
func ServiceRestart(n remotessh.TestbedNode, srv string) (string, error) {
return n.RunCommandWithOutput(fmt.Sprintf("sudo systemctl restart %s", srv))
}
func (s *systemtestSuite) clearVolumeHost(node remotessh.TestbedNode) error {
logrus.Infof("Clearing volumes on %q", node.GetName())
node.RunCommand("docker volume ls | tail -n +2 | awk '{ print $2 }' | xargs docker volume rm")
return nil
}
func ClearEtcd(node remotessh.TestbedNode) {
logrus.Info("Waiting for etcd")
node.RunCommand("while ! $(etcdctl cluster-health | tail -1 | grep -q 'cluster is healthy'); do sleep 1; done")
logrus.Info("Clearing etcd data")
node.RunCommand("etcdctl rm --recursive /volplugin")
}
func startVolplugin(node remotessh.TestbedNode) error {
logrus.Infof("Starting the volplugin on %q", node.GetName())
return node.RunCommandBackground("sudo systemctl start volplugin")
}
func stopAPIServer(node remotessh.TestbedNode) error {
logrus.Infof("Stopping the apiserver on %q", node.GetName())
defer time.Sleep(time.Second)
return node.RunCommand("sudo systemctl stop apiserver")
}
func startAPIServer(node remotessh.TestbedNode) error {
logrus.Infof("Starting the apiserver on %q", node.GetName())
err := node.RunCommandBackground("sudo systemctl start apiserver")
logrus.Infof("Waiting for apiserver startup on %q", node.GetName())
return err
}
func ClearEtcd(node remotessh.TestbedNode) {
log.Infof("Clearing etcd data")
node.RunCommand(`for i in $(etcdctl ls /); do etcdctl rm --recursive "$i"; done`)
}
//ServiceStatus queries and returns status result of systemd service unit
func ServiceStatus(n remotessh.TestbedNode, srv string) (string, error) {
return n.RunCommandWithOutput(fmt.Sprintf("systemctl status %s", srv))
}
func stopVolplugin(node remotessh.TestbedNode) error {
logrus.Infof("Stopping the volplugin on %q", node.GetName())
defer time.Sleep(time.Second)
return node.RunCommand("sudo systemctl stop volplugin")
}