// StartClientWithEnvAndArgs starts a client container with specified env-variables.
// It expects ping to server container to succeed
func StartClientWithEnvAndArgs(t *testing.T, node stu.TestbedNode, contName, ipAddress string,
env, dockerArgs []string) {
cmdStr := "sudo %s docker run %s --name=" + contName +
" ubuntu /bin/bash -c \"ping -c5 " + ipAddress + "\""
cmdStr = fmt.Sprintf(cmdStr, strings.Join(env, " "),
strings.Join(dockerArgs, " "))
output, err := node.RunCommandWithOutput(cmdStr)
if err != nil {
OvsDumpInfo(node)
t.Fatalf("Error '%s' launching container '%s', Output: \n%s\n",
err, contName, output)
}
cmdStr = fmt.Sprintf("sudo docker logs %s", contName)
output, err = node.RunCommandWithOutput(cmdStr)
if err != nil {
t.Fatalf("Error '%s' fetching container '%s' logs, Output: \n%s\n",
err, contName, output)
}
//verify that the output indicates <100% loss (some loss is expected due to
// timing of interface creation and starting ping)
if strings.Contains(string(output), ", 100% packet loss,") {
OvsDumpInfo(node)
t.Fatalf("Ping test failed for container '%s', Output: \n%s\n",
contName, output)
}
}
// GetIPAddress returns IP-address information for specified endpoint
func GetIPAddress(t *testing.T, node stu.TestbedNode, ep, stateStore string) string {
cmdStr := "netdcli -oper get -construct endpoint " + ep + " 2>&1"
if stateStore != "" {
cmdStr = "netdcli -oper get -state-store " + stateStore + " -construct endpoint " + ep + " 2>&1"
}
output, err := node.RunCommandWithOutput(cmdStr)
if err != nil || string(output) == "" {
time.Sleep(2 * time.Second)
output, err = node.RunCommandWithOutput(cmdStr)
if err != nil || output == "" {
t.Fatalf("Error getting ip for ep %s. Error: %s, Cmdstr: %s, Output: \n%s\n",
err, ep, cmdStr, output)
}
}
output = strings.Trim(string(output), "[]")
epStruct := drivers.OvsOperEndpointState{}
if err := json.Unmarshal([]byte(output), &epStruct); err != nil {
t.Fatalf("Error getting ip for ep %s. Error: %s, Cmdstr: %s, Output: \n%s\n",
err, ep, cmdStr, output)
}
return epStruct.IPAddress
}
func startVolmaster(node utils.TestbedNode) error {
log.Infof("Starting the volmaster on %s", node.GetName())
_, err := node.RunCommandBackground("sudo -E nohup `which volmaster` --debug </dev/null &>/tmp/volmaster.log &")
log.Infof("Waiting for volmaster startup")
time.Sleep(10 * time.Millisecond)
return err
}
// StartNetmasterWithFlags starts netplugin on specified testbed nodes with specified flags
func StartNetmasterWithFlags(t *testing.T, node stu.TestbedNode, flags map[string]string) {
time.Sleep(5 * time.Second)
var (
cmdStr string
flagsStr string
)
for k, v := range flags {
flagsStr += fmt.Sprintf("%s=%s", k, v)
}
if os.Getenv("CONTIV_TESTBED") == "DIND" {
cmdStr = fmt.Sprintf("netmaster %s 1>/tmp/netmaster.log 2>&1", flagsStr)
} else {
cmdStr = fmt.Sprintf("nohup netmaster %s 0<&- &>/tmp/netmaster.log", flagsStr)
}
output, err := node.RunCommandBackground(cmdStr)
if err != nil {
t.Fatalf("Failed to launch netplugin. Error: %s\nCmd:%q\n Output : %s\n",
err, cmdStr, output)
}
time.Sleep(5 * time.Second)
}
// DumpNetpluginLogs prints netplugin logs from the specified testbed node
func DumpNetpluginLogs(node stu.TestbedNode) {
cmdStr := fmt.Sprintf("sudo cat /tmp/netplugin.log")
output, err := node.RunCommandWithOutput(cmdStr)
if err == nil {
log.Debugf("logs on node %s: \n%s\n", node.GetName(), output)
}
}
func restartDockerHost(node utils.TestbedNode) error {
log.Infof("Restarting docker on %q", node.GetName())
// note that for all these restart tasks we error out quietly to avoid other
// hosts being cleaned up
node.RunCommand("sudo service docker restart")
return nil
}
func getContainerUUID(node stu.TestbedNode, contName string) (string, error) {
cmdStr := "sudo docker inspect --format='{{.Id}}' " + contName
output, err := node.RunCommandWithOutput(cmdStr)
if err != nil {
output = ""
}
return strings.TrimSpace(output), err
}
// DockerCleanupWithEnv kills and removes a container on a specified testbed node
// and with specified env-variables
func DockerCleanupWithEnv(t *testing.T, node stu.TestbedNode, contName string, env []string) {
if !OkToCleanup(t.Failed()) {
return
}
cmdStr := fmt.Sprintf("sudo %s docker kill %s", strings.Join(env, " "), contName)
node.RunCommand(cmdStr)
cmdStr = fmt.Sprintf("sudo %s docker rm %s", strings.Join(env, " "), contName)
node.RunCommand(cmdStr)
}
func startVolplugin(node utils.TestbedNode) error {
log.Infof("Starting the volplugin on %q", node.GetName())
defer time.Sleep(10 * time.Millisecond)
// FIXME this is hardcoded because it's simpler. If we move to
// multimaster or change the monitor subnet, we will have issues.
_, err := node.RunCommandBackground("sudo -E `which volplugin` --debug &>/tmp/volplugin.log &")
return err
}
// StartServerWithEnvAndArgs starts a server container with specified env-variables
func StartServerWithEnvAndArgs(t *testing.T, node stu.TestbedNode, contName string,
env, dockerArgs []string) {
cmdStr := "sudo %s docker run -d %s --name=" + contName +
" ubuntu /bin/bash -c \"mkfifo foo && < foo\""
cmdStr = fmt.Sprintf(cmdStr, strings.Join(env, " "),
strings.Join(dockerArgs, " "))
output, err := node.RunCommandWithOutput(cmdStr)
if err != nil {
OvsDumpInfo(node)
t.Fatalf("Error '%s' launching container '%s', Output: \n%s\n",
err, contName, output)
}
}
// NetworkStateExists tests if state for specified network exists
func NetworkStateExists(node stu.TestbedNode, network, stateStore string) error {
cmdStr := "netdcli -oper get -construct network " + network + " 2>&1"
if stateStore != "" {
cmdStr = "netdcli -state-store " + stateStore + "-oper get -construct network " + network + " 2>&1"
}
output, err := node.RunCommandWithOutput(cmdStr)
if err != nil {
return err
}
if string(output) == "" {
return core.Errorf("got null output")
}
return nil
}
// StartClientFailureWithEnvAndArgs starts a client container with specified env-variables.
// It expects ping to server container to failure
func StartClientFailureWithEnvAndArgs(t *testing.T, node stu.TestbedNode, contName, ipAddress string,
env, dockerArgs []string) {
cmdStr := "sudo %s docker run %s --name=" + contName +
" ubuntu /bin/bash -c \"ping -c5 " + ipAddress + "\""
cmdStr = fmt.Sprintf(cmdStr, strings.Join(env, " "),
strings.Join(dockerArgs, " "))
output, err := node.RunCommandWithOutput(cmdStr)
if err == nil {
t.Fatalf("Ping did not fail as expected, err '%s' container '%s', "+
"Output: \n%s\n", err, contName, output)
}
cmdStr = fmt.Sprintf("sudo docker logs %s", contName)
output, err = node.RunCommandWithOutput(cmdStr)
if err != nil || !strings.Contains(string(output), ", 100% packet loss,") {
t.Fatalf("Ping did not fail as expected, err '%s' container '%s', "+
"Output: \n%s\n", err, contName, output)
}
}
func applyConfig(t *testing.T, cfgType, jsonCfg string, node stu.TestbedNode, stateStore string) {
// replace newlines with space and "(quote) with \"(escaped quote) for
// echo to consume and produce desired json config
jsonCfg = getEchoCompatibleStr(jsonCfg)
cmdStr := fmt.Sprintf("echo \"%s\" > /tmp/netdcli.cfg", jsonCfg)
output, err := node.RunCommandWithOutput(cmdStr)
if err != nil {
t.Fatalf("Error '%s' creating config file\nCmd: %q\n Output : %s \n",
err, cmdStr, output)
}
cmdStr = "netdcli -" + cfgType + " /tmp/netdcli.cfg 2>&1"
if stateStore != "" {
cmdStr = "netdcli -state-store " + stateStore + " -" + cfgType + " /tmp/netdcli.cfg 2>&1"
}
output, err = node.RunCommandWithOutput(cmdStr)
if err != nil {
t.Fatalf("Failed to apply config. Error: %s\nCmd: %q\n Output : %s\n",
err, cmdStr, output)
}
}
func stopVolplugin(node utils.TestbedNode) error {
log.Infof("Stopping the volplugin on %q", node.GetName())
return node.RunCommand("sudo pkill volplugin")
}
func stopVolmaster(node utils.TestbedNode) error {
log.Infof("Stopping the volmaster on %s", node.GetName())
return node.RunCommand("sudo pkill volmaster")
}
func startVolsupervisor(node utils.TestbedNode) error {
log.Infof("Starting the volsupervisor on %s", node.GetName())
_, err := node.RunCommandBackground("sudo -E nohup `which volsupervisor` --debug </dev/null &>/tmp/volsupervisor.log &")
return err
}
func (s *systemtestSuite) volpluginStop(node utils.TestbedNode) error {
log.Infof("Stopping the volplugin on %q", node.GetName())
return node.RunCommand("sudo pkill volplugin")
}
func volpluginStop(node utils.TestbedNode) error {
return node.RunCommand("sudo pkill volplugin")
}
func (s *systemtestSuite) clearContainerHost(node utils.TestbedNode) error {
log.Infof("Clearing containers on %q", node.GetName())
node.RunCommand("docker ps -aq | xargs docker rm -f")
return nil
}
// OvsDumpInfo dumps the ovs state on the specified testbed node
func OvsDumpInfo(node stu.TestbedNode) {
cmdStr := "sudo ovs-vsctl show"
output, _ := node.RunCommandWithOutput(cmdStr)
log.Debugf("ovs-vsctl on node %s: \n%s\n", node.GetName(), output)
}
// StopNetmaster stops the netmaster on specified testbed node
func StopNetmaster(t *testing.T, node stu.TestbedNode) {
cmdStr := "sudo pkill netmaster"
node.RunCommand(cmdStr)
}
func (s *systemtestSuite) clearVolumeHost(node utils.TestbedNode) error {
log.Infof("Clearing volumes on %q", node.GetName())
node.RunCommand("docker volume ls | tail -n +2 | awk '{ print $2 }' | xargs docker volume rm")
return nil
}
func volpluginStart(node utils.TestbedNode) error {
// FIXME this is hardcoded because it's simpler. If we move to
// multimaster or change the monitor subnet, we will have issues.
_, err := node.RunCommandBackground("sudo -E `which volplugin` --debug --master 192.168.24.10:8080 tenant1 &>/tmp/volplugin.log &")
return err
}