// Instantiates a Framework. Cleanup *must* be called. Class is thread-compatible.
// All framework actions report fatal errors on the t specified at creation time.
//
// Typical use:
//
// func TestFoo(t *testing.T) {
// fm := framework.New(t)
// defer fm.Cleanup()
// ... actual test ...
// }
func New(t *testing.T) Framework {
// All integration tests are large.
if testing.Short() {
t.Skip("Skipping framework test in short mode")
}
// Try to see if non-localhost hosts are GCE instances.
var gceInstanceName string
hostname := *host
if hostname != "localhost" {
gceInstanceName = hostname
gceIp, err := common.GetGceIp(hostname)
if err == nil {
hostname = gceIp
}
}
fm := &realFramework{
hostname: HostnameInfo{
Host: hostname,
Port: *port,
GceInstanceName: gceInstanceName,
},
t: t,
cleanups: make([]func(), 0),
}
fm.shellActions = shellActions{
fm: fm,
}
fm.dockerActions = dockerActions{
fm: fm,
}
return fm
}
func PushAndRunTests(host, testDir string) error {
// Push binary.
glog.Infof("Pushing cAdvisor binary to %q...", host)
args := common.GetGCComputeArgs("ssh", host, "--", "mkdir", "-p", testDir)
err := RunCommand("gcloud", args...)
if err != nil {
return fmt.Errorf("failed to make remote testing directory: %v", err)
}
defer func() {
args := common.GetGCComputeArgs("ssh", host, "--", "rm", "-rf", testDir)
err := RunCommand("gcloud", args...)
if err != nil {
glog.Errorf("Failed to cleanup test directory: %v", err)
}
}()
args = common.GetGCComputeArgs("copy-files", cadvisorBinary, fmt.Sprintf("%s:%s", host, testDir))
err = RunCommand("gcloud", args...)
if err != nil {
return fmt.Errorf("failed to copy binary: %v", err)
}
// TODO(vmarmol): Get logs in case of failures.
// Start cAdvisor.
glog.Infof("Running cAdvisor on %q...", host)
portStr := strconv.Itoa(*port)
errChan := make(chan error)
go func() {
args = common.GetGCComputeArgs("ssh", host, "--", fmt.Sprintf("sudo %s --port %s --logtostderr", path.Join(testDir, cadvisorBinary), portStr))
err = RunCommand("gcloud", args...)
if err != nil {
errChan <- fmt.Errorf("error running cAdvisor: %v", err)
}
}()
defer func() {
args = common.GetGCComputeArgs("ssh", host, "--", "sudo", "pkill", cadvisorBinary)
err := RunCommand("gcloud", args...)
if err != nil {
glog.Errorf("Failed to cleanup: %v", err)
}
}()
ipAddress, err := common.GetGceIp(host)
if err != nil {
return fmt.Errorf("failed to get GCE IP: %v", err)
}
// Wait for cAdvisor to come up.
endTime := time.Now().Add(*cadvisorTimeout)
done := false
for endTime.After(time.Now()) && !done {
select {
case err := <-errChan:
// Quit early if there was an error.
return err
case <-time.After(500 * time.Millisecond):
// Stop waiting when cAdvisor is healthy..
resp, err := http.Get(fmt.Sprintf("http://%s:%s/healthz", ipAddress, portStr))
if err == nil && resp.StatusCode == http.StatusOK {
done = true
break
}
}
}
if !done {
return fmt.Errorf("timed out waiting for cAdvisor to come up at host %q", host)
}
// Run the tests.
glog.Infof("Running integration tests targeting %q...", host)
err = RunCommand("godep", "go", "test", "github.com/google/cadvisor/integration/tests/...", "--host", host, "--port", portStr)
if err != nil {
return err
}
return nil
}
func PushAndRunTests(host, testDir string) error {
// Push binary.
glog.Infof("Pushing cAdvisor binary to %q...", host)
args := common.GetGCComputeArgs("ssh", host, "--", "mkdir", "-p", testDir)
err := RunCommand("gcloud", args...)
if err != nil {
return fmt.Errorf("failed to make remote testing directory: %v", err)
}
defer func() {
args := common.GetGCComputeArgs("ssh", host, "--", "rm", "-rf", testDir)
err := RunCommand("gcloud", args...)
if err != nil {
glog.Errorf("Failed to cleanup test directory: %v", err)
}
}()
args = common.GetGCComputeArgs("copy-files", cadvisorBinary, fmt.Sprintf("%s:%s", host, testDir))
err = RunCommand("gcloud", args...)
if err != nil {
return fmt.Errorf("failed to copy binary: %v", err)
}
// Start cAdvisor.
glog.Infof("Running cAdvisor on %q...", host)
portStr := strconv.Itoa(*port)
errChan := make(chan error)
go func() {
args = common.GetGCComputeArgs("ssh", host, "--", fmt.Sprintf("sudo %s --port %s --logtostderr &> %s/log.txt", path.Join(testDir, cadvisorBinary), portStr, testDir))
err = RunCommand("gcloud", args...)
if err != nil {
errChan <- fmt.Errorf("error running cAdvisor: %v", err)
}
}()
defer func() {
args = common.GetGCComputeArgs("ssh", host, "--", "sudo", "pkill", cadvisorBinary)
err := RunCommand("gcloud", args...)
if err != nil {
glog.Errorf("Failed to cleanup: %v", err)
}
}()
ipAddress, err := common.GetGceIp(host)
if err != nil {
return fmt.Errorf("failed to get GCE IP: %v", err)
}
// Wait for cAdvisor to come up.
endTime := time.Now().Add(*cadvisorTimeout)
done := false
for endTime.After(time.Now()) && !done {
select {
case err := <-errChan:
// Quit early if there was an error.
return err
case <-time.After(500 * time.Millisecond):
// Stop waiting when cAdvisor is healthy..
resp, err := http.Get(fmt.Sprintf("http://%s:%s/healthz", ipAddress, portStr))
if err == nil && resp.StatusCode == http.StatusOK {
done = true
break
}
}
}
if !done {
return fmt.Errorf("timed out waiting for cAdvisor to come up at host %q", host)
}
// Get attributes for debugging purposes.
attributes, err := getAttributes(ipAddress, portStr)
if err != nil {
return fmt.Errorf("%v - %q", err, host)
}
// Run the tests in a retry loop.
glog.Infof("Running integration tests targeting %q...", host)
for i := 0; i <= *testRetryCount; i++ {
// Check if this is a retry
if i > 0 {
time.Sleep(time.Second * 15) // Wait 15 seconds before retrying
glog.Warningf("Retrying (%d of %d) tests on host %s due to error %v", i, *testRetryCount, host, err)
}
// Run the command
err = RunCommand("godep", "go", "test", "github.com/google/cadvisor/integration/tests/...", "--host", host, "--port", portStr)
if err == nil {
// On success, break out of retry loop
break
}
// Only retry on test failures caused by these known flaky failure conditions
if retryRegex == nil || !retryRegex.Match([]byte(err.Error())) {
glog.Warningf("Skipping retry for tests on host %s because error is not whitelisted: %s", host, err.Error())
break
}
}
if err != nil {
// Copy logs from the host
args = common.GetGCComputeArgs("copy-files", fmt.Sprintf("%s:%s/log.txt", host, testDir), "./")
// Declare new error or it will get shadowed by logs, err := <> and we won't be able to unset it from nil
err2 := RunCommand("gcloud", args...)
if err2 != nil {
return fmt.Errorf("error fetching logs: %v for %v", err2, err)
}
defer os.Remove("./log.txt")
logs, err2 := ioutil.ReadFile("./log.txt")
if err2 != nil {
return fmt.Errorf("error reading local log file: %v for %v", err2, err)
}
glog.Errorf("----------------------\nLogs from Host: %q\n%v\n", host, string(logs))
err = fmt.Errorf("error on host %s: %v\n%+v", host, err, attributes)
}
return err
}