func (self shellActions) RunStress(command string, args ...string) (string, string) { var cmd *exec.Cmd if self.fm.Hostname().Host == "localhost" { // Just run locally. cmd = exec.Command(command, args...) } else { // We must SSH to the remote machine and run the command. args = append(common.GetGCComputeArgs("ssh", self.fm.Hostname().GceInstanceName, "--", command), args...) cmd = exec.Command("gcloud", args...) } var stdout bytes.Buffer var stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr err := cmd.Run() if err != nil { self.fm.T().Logf("Ran %q %v in %q and received error: %q. Stdout: %q, Stderr: %s", command, args, self.fm.Hostname().Host, err, stdout.String(), stderr.String()) return stdout.String(), stderr.String() } return stdout.String(), stderr.String() }
func PushAndRunTests(host, testDir string) error { // Push binary. glog.Infof("Pushing cAdvisor binary to %q...", host) args := common.GetGCComputeArgs("ssh", host, "--", "mkdir", "-p", testDir) err := RunCommand("gcloud", args...) if err != nil { return fmt.Errorf("failed to make remote testing directory: %v", err) } defer func() { args := common.GetGCComputeArgs("ssh", host, "--", "rm", "-rf", testDir) err := RunCommand("gcloud", args...) if err != nil { glog.Errorf("Failed to cleanup test directory: %v", err) } }() args = common.GetGCComputeArgs("copy-files", cadvisorBinary, fmt.Sprintf("%s:%s", host, testDir)) err = RunCommand("gcloud", args...) if err != nil { return fmt.Errorf("failed to copy binary: %v", err) } // TODO(vmarmol): Get logs in case of failures. // Start cAdvisor. glog.Infof("Running cAdvisor on %q...", host) portStr := strconv.Itoa(*port) errChan := make(chan error) go func() { args = common.GetGCComputeArgs("ssh", host, "--", fmt.Sprintf("sudo %s --port %s --logtostderr", path.Join(testDir, cadvisorBinary), portStr)) err = RunCommand("gcloud", args...) if err != nil { errChan <- fmt.Errorf("error running cAdvisor: %v", err) } }() defer func() { args = common.GetGCComputeArgs("ssh", host, "--", "sudo", "pkill", cadvisorBinary) err := RunCommand("gcloud", args...) if err != nil { glog.Errorf("Failed to cleanup: %v", err) } }() ipAddress, err := common.GetGceIp(host) if err != nil { return fmt.Errorf("failed to get GCE IP: %v", err) } // Wait for cAdvisor to come up. endTime := time.Now().Add(*cadvisorTimeout) done := false for endTime.After(time.Now()) && !done { select { case err := <-errChan: // Quit early if there was an error. return err case <-time.After(500 * time.Millisecond): // Stop waiting when cAdvisor is healthy.. resp, err := http.Get(fmt.Sprintf("http://%s:%s/healthz", ipAddress, portStr)) if err == nil && resp.StatusCode == http.StatusOK { done = true break } } } if !done { return fmt.Errorf("timed out waiting for cAdvisor to come up at host %q", host) } // Run the tests. glog.Infof("Running integration tests targeting %q...", host) err = RunCommand("godep", "go", "test", "github.com/google/cadvisor/integration/tests/...", "--host", host, "--port", portStr) if err != nil { return err } return nil }
func PushAndRunTests(host, testDir string) error { // Push binary. glog.Infof("Pushing cAdvisor binary to %q...", host) args := common.GetGCComputeArgs("ssh", host, "--", "mkdir", "-p", testDir) err := RunCommand("gcloud", args...) if err != nil { return fmt.Errorf("failed to make remote testing directory: %v", err) } defer func() { args := common.GetGCComputeArgs("ssh", host, "--", "rm", "-rf", testDir) err := RunCommand("gcloud", args...) if err != nil { glog.Errorf("Failed to cleanup test directory: %v", err) } }() args = common.GetGCComputeArgs("copy-files", cadvisorBinary, fmt.Sprintf("%s:%s", host, testDir)) err = RunCommand("gcloud", args...) if err != nil { return fmt.Errorf("failed to copy binary: %v", err) } // Start cAdvisor. glog.Infof("Running cAdvisor on %q...", host) portStr := strconv.Itoa(*port) errChan := make(chan error) go func() { args = common.GetGCComputeArgs("ssh", host, "--", fmt.Sprintf("sudo %s --port %s --logtostderr &> %s/log.txt", path.Join(testDir, cadvisorBinary), portStr, testDir)) err = RunCommand("gcloud", args...) if err != nil { errChan <- fmt.Errorf("error running cAdvisor: %v", err) } }() defer func() { args = common.GetGCComputeArgs("ssh", host, "--", "sudo", "pkill", cadvisorBinary) err := RunCommand("gcloud", args...) if err != nil { glog.Errorf("Failed to cleanup: %v", err) } }() ipAddress, err := common.GetGceIp(host) if err != nil { return fmt.Errorf("failed to get GCE IP: %v", err) } // Wait for cAdvisor to come up. endTime := time.Now().Add(*cadvisorTimeout) done := false for endTime.After(time.Now()) && !done { select { case err := <-errChan: // Quit early if there was an error. return err case <-time.After(500 * time.Millisecond): // Stop waiting when cAdvisor is healthy.. resp, err := http.Get(fmt.Sprintf("http://%s:%s/healthz", ipAddress, portStr)) if err == nil && resp.StatusCode == http.StatusOK { done = true break } } } if !done { return fmt.Errorf("timed out waiting for cAdvisor to come up at host %q", host) } // Get attributes for debugging purposes. attributes, err := getAttributes(ipAddress, portStr) if err != nil { return fmt.Errorf("%v - %q", err, host) } // Run the tests in a retry loop. glog.Infof("Running integration tests targeting %q...", host) for i := 0; i <= *testRetryCount; i++ { // Check if this is a retry if i > 0 { time.Sleep(time.Second * 15) // Wait 15 seconds before retrying glog.Warningf("Retrying (%d of %d) tests on host %s due to error %v", i, *testRetryCount, host, err) } // Run the command err = RunCommand("godep", "go", "test", "github.com/google/cadvisor/integration/tests/...", "--host", host, "--port", portStr) if err == nil { // On success, break out of retry loop break } // Only retry on test failures caused by these known flaky failure conditions if retryRegex == nil || !retryRegex.Match([]byte(err.Error())) { glog.Warningf("Skipping retry for tests on host %s because error is not whitelisted: %s", host, err.Error()) break } } if err != nil { // Copy logs from the host args = common.GetGCComputeArgs("copy-files", fmt.Sprintf("%s:%s/log.txt", host, testDir), "./") // Declare new error or it will get shadowed by logs, err := <> and we won't be able to unset it from nil err2 := RunCommand("gcloud", args...) if err2 != nil { return fmt.Errorf("error fetching logs: %v for %v", err2, err) } defer os.Remove("./log.txt") logs, err2 := ioutil.ReadFile("./log.txt") if err2 != nil { return fmt.Errorf("error reading local log file: %v for %v", err2, err) } glog.Errorf("----------------------\nLogs from Host: %q\n%v\n", host, string(logs)) err = fmt.Errorf("error on host %s: %v\n%+v", host, err, attributes) } return err }