func TestOVirtCloudConfiguration(t *testing.T) { config1 := (io.Reader)(nil) _, err1 := cloudprovider.GetCloudProvider("ovirt", config1) if err1 == nil { t.Fatalf("An error is expected when the configuration is missing") } config2 := strings.NewReader("") _, err2 := cloudprovider.GetCloudProvider("ovirt", config2) if err2 == nil { t.Fatalf("An error is expected when the configuration is empty") } config3 := strings.NewReader(` [connection] `) _, err3 := cloudprovider.GetCloudProvider("ovirt", config3) if err3 == nil { t.Fatalf("An error is expected when the uri is missing") } config4 := strings.NewReader(` [connection] uri = https://localhost:8443/ovirt-engine/api `) _, err4 := cloudprovider.GetCloudProvider("ovirt", config4) if err4 != nil { t.Fatalf("Unexpected error creating the provider: %s", err4) } }
// NewClusterManager creates a cluster manager for shared resources. // - name: is the name used to tag cluster wide shared resources. This is the // string passed to glbc via --gce-cluster-name. // - defaultBackendNodePort: is the node port of glbc's default backend. This is // the kubernetes Service that serves the 404 page if no urls match. // - defaultHealthCheckPath: is the default path used for L7 health checks, eg: "/healthz" func NewClusterManager( name string, defaultBackendNodePort int64, defaultHealthCheckPath string) (*ClusterManager, error) { cloudInterface, err := cloudprovider.GetCloudProvider("gce", nil) if err != nil { return nil, err } cloud := cloudInterface.(*gce.GCECloud) cluster := ClusterManager{ClusterNamer: utils.Namer{name}} zone, err := cloud.GetZone() if err != nil { return nil, err } cluster.instancePool = instances.NewNodePool(cloud, zone.FailureDomain) healthChecker := healthchecks.NewHealthChecker(cloud, defaultHealthCheckPath, cluster.ClusterNamer) cluster.backendPool = backends.NewBackendPool( cloud, healthChecker, cluster.instancePool, cluster.ClusterNamer) defaultBackendHealthChecker := healthchecks.NewHealthChecker(cloud, "/healthz", cluster.ClusterNamer) defaultBackendPool := backends.NewBackendPool( cloud, defaultBackendHealthChecker, cluster.instancePool, cluster.ClusterNamer) cluster.defaultBackendNodePort = defaultBackendNodePort cluster.l7Pool = loadbalancers.NewLoadBalancerPool( cloud, defaultBackendPool, defaultBackendNodePort, cluster.ClusterNamer) return &cluster, nil }
func getGCEClient(config io.Reader) *gce.GCECloud { // Creating the cloud interface involves resolving the metadata server to get // an oauth token. If this fails, the token provider assumes it's not on GCE. // No errors are thrown. So we need to keep retrying till it works because // we know we're on GCE. for { cloudInterface, err := cloudprovider.GetCloudProvider("gce", config) if err == nil { cloud := cloudInterface.(*gce.GCECloud) // If this controller is scheduled on a node without compute/rw // it won't be allowed to list backends. We can assume that the // user has no need for Ingress in this case. If they grant // permissions to the node they will have to restart the controller // manually to re-create the client. if _, err = cloud.ListBackendServices(); err == nil || utils.IsHTTPErrorCode(err, http.StatusForbidden) { return cloud } glog.Warningf("Failed to list backend services, retrying: %v", err) } else { glog.Warningf("Failed to retrieve cloud interface, retrying: %v", err) } time.Sleep(cloudClientRetryInterval) } }
// Return cloud provider func getCloudProvider() (*aws.AWSCloud, error) { awsCloudProvider, err := cloudprovider.GetCloudProvider("aws", nil) if err != nil || awsCloudProvider == nil { return nil, err } // The conversion must be safe otherwise bug in GetCloudProvider() return awsCloudProvider.(*aws.AWSCloud), nil }
// Return cloud provider func getCloudProvider() (*gcecloud.GCECloud, error) { gceCloudProvider, err := cloudprovider.GetCloudProvider("gce", nil) if err != nil || gceCloudProvider == nil { return nil, err } // The conversion must be safe otherwise bug in GetCloudProvider() return gceCloudProvider.(*gcecloud.GCECloud), nil }
func TestE2E(t *testing.T) { util.ReallyCrash = true util.InitLogs() defer util.FlushLogs() if *reportDir != "" { if err := os.MkdirAll(*reportDir, 0755); err != nil { glog.Errorf("Failed creating report directory: %v", err) } defer CoreDump(*reportDir) } if testContext.Provider == "" { glog.Info("The --provider flag is not set. Treating as a conformance test. Some tests may not be run.") } if testContext.Provider == "aws" { awsConfig := "[Global]\n" if cloudConfig.Zone == "" { glog.Fatal("gce-zone must be specified for AWS") } awsConfig += fmt.Sprintf("Zone=%s\n", cloudConfig.Zone) if cloudConfig.ClusterTag == "" { glog.Fatal("--cluster-tag must be specified for AWS") } awsConfig += fmt.Sprintf("KubernetesClusterTag=%s\n", cloudConfig.ClusterTag) var err error cloudConfig.Provider, err = cloudprovider.GetCloudProvider(testContext.Provider, strings.NewReader(awsConfig)) if err != nil { glog.Fatal("Error building AWS provider: ", err) } } // Disable skipped tests unless they are explicitly requested. if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" { config.GinkgoConfig.SkipString = "Skipped" } gomega.RegisterFailHandler(ginkgo.Fail) // Ensure all pods are running and ready before starting tests (otherwise, // cluster infrastructure pods that are being pulled or started can block // test pods from running, and tests that ensure all pods are running and // ready will fail). if err := waitForPodsRunningReady(api.NamespaceSystem, testContext.MinStartupPods, podStartupTimeout); err != nil { t.Errorf("Error waiting for all pods to be running and ready: %v", err) return } // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins var r []ginkgo.Reporter if *reportDir != "" { r = append(r, reporters.NewJUnitReporter(path.Join(*reportDir, fmt.Sprintf("junit_%02d.xml", config.GinkgoConfig.ParallelNode)))) } ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes e2e suite", r) }
// getVolumeProvider returns the AWS Volumes interface func (ebs *awsElasticBlockStore) getVolumeProvider() (aws_cloud.Volumes, error) { name := "aws" cloud, err := cloudprovider.GetCloudProvider(name, nil) if err != nil { return nil, err } volumes, ok := cloud.(aws_cloud.Volumes) if !ok { return nil, fmt.Errorf("Cloud provider does not support volumes") } return volumes, nil }
// NewClusterManager creates a cluster manager for shared resources. // - name: is the name used to tag cluster wide shared resources. This is the // string passed to glbc via --gce-cluster-name. // - defaultBackendNodePort: is the node port of glbc's default backend. This is // the kubernetes Service that serves the 404 page if no urls match. // - defaultHealthCheckPath: is the default path used for L7 health checks, eg: "/healthz" func NewClusterManager(name string, defaultBackendNodePort int64, defaultHealthCheckPath string) (*ClusterManager, error) { cloudInterface, err := cloudprovider.GetCloudProvider("gce", nil) if err != nil { return nil, err } cloud := cloudInterface.(*gce.GCECloud) cluster := ClusterManager{ClusterName: name} // Why do we need so many defaults? // Default IG: We add all instances to a single ig, and // every service that requires loadbalancing opens up // a nodePort on the cluster, which translates to a node // on this default ig. // // Default Backend: We need a default backend to create // every urlmap, even if the user doesn't specify one. // This is the backend that gets requests if no paths match. // Note that this backend doesn't actually occupy a port // on the instance group. // // Default Health Check: The default backend used by an // Ingress that doesn't specify it. defaultIGName := defaultInstanceGroupName(name) if cluster.instancePool, err = NewNodePool(cloud, defaultIGName); err != nil { return nil, err } // TODO: We're roud tripping for a resource we just created. defaultIG, err := cluster.instancePool.Get(defaultIGName) if err != nil { return nil, err } if cluster.healthChecker, err = NewHealthChecker( cloud, defaultHttpHealthCheck, defaultHealthCheckPath); err != nil { return nil, err } defaultHc, err := cluster.healthChecker.Get(defaultHttpHealthCheck) if err != nil { return nil, err } if cluster.backendPool, err = NewBackendPool( cloud, defaultBackendNodePort, defaultIG, defaultHc, cloud); err != nil { return nil, err } cluster.defaultBackendNodePort = defaultBackendNodePort // TODO: Don't cast, the problem here is the default backend doesn't have // a port and the interface only allows backend access via port. cluster.l7Pool = NewLoadBalancerPool( cloud, cluster.backendPool.(*Backends).defaultBackend) return &cluster, nil }
// Attaches the specified persistent disk device to node, verifies that it is attached, and retries if it fails. func attachDiskAndVerify(b *gcePersistentDiskBuilder, sdBeforeSet sets.String) (string, error) { devicePaths := getDiskByIdPaths(b.gcePersistentDisk) var gce cloudprovider.Interface for numRetries := 0; numRetries < maxRetries; numRetries++ { if gce == nil { var err error gce, err = cloudprovider.GetCloudProvider("gce", nil) if err != nil || gce == nil { // Retry on error. See issue #11321 glog.Errorf("Error getting GCECloudProvider while attaching PD %q: %v", b.pdName, err) gce = nil time.Sleep(errorSleepDuration) continue } } if numRetries > 0 { glog.Warningf("Timed out waiting for GCE PD %q to attach. Retrying attach.", b.pdName) } if err := gce.(*gce_cloud.GCECloud).AttachDisk(b.pdName, b.readOnly); err != nil { // Retry on error. See issue #11321. Continue and verify if disk is attached, because a // previous attach operation may still succeed. glog.Errorf("Error attaching PD %q: %v", b.pdName, err) } for numChecks := 0; numChecks < maxChecks; numChecks++ { if err := udevadmChangeToNewDrives(sdBeforeSet); err != nil { // udevadm errors should not block disk attachment, log and continue glog.Errorf("%v", err) } for _, path := range devicePaths { if pathExists, err := pathExists(path); err != nil { // Retry on error. See issue #11321 glog.Errorf("Error checking if path exists: %v", err) } else if pathExists { // A device path has successfully been created for the PD glog.Infof("Successfully attached GCE PD %q.", b.pdName) return path, nil } } // Sleep then check again glog.V(3).Infof("Waiting for GCE PD %q to attach.", b.pdName) time.Sleep(checkSleepDuration) } } return "", fmt.Errorf("Could not attach GCE PD %q. Timeout waiting for mount paths to be created.", b.pdName) }
// setupProviderConfig validates and sets up cloudConfig based on testContext.Provider. func setupProviderConfig() error { switch testContext.Provider { case "": glog.Info("The --provider flag is not set. Treating as a conformance test. Some tests may not be run.") case "gce", "gke": var err error Logf("Fetching cloud provider for %q\r\n", testContext.Provider) var tokenSource oauth2.TokenSource tokenSource = nil if cloudConfig.ServiceAccount != "" { // Use specified service account for auth Logf("Using service account %q as token source.", cloudConfig.ServiceAccount) tokenSource = google.ComputeTokenSource(cloudConfig.ServiceAccount) } zone := testContext.CloudConfig.Zone region, err := gcecloud.GetGCERegion(zone) if err != nil { return fmt.Errorf("error parsing GCE/GKE region from zone %q: %v", zone, err) } managedZones := []string{zone} // Only single-zone for now cloudConfig.Provider, err = gcecloud.CreateGCECloud(testContext.CloudConfig.ProjectID, region, zone, managedZones, "" /* networkUrl */, tokenSource, false /* useMetadataServer */) if err != nil { return fmt.Errorf("Error building GCE/GKE provider: %v", err) } case "aws": awsConfig := "[Global]\n" if cloudConfig.Zone == "" { return fmt.Errorf("gce-zone must be specified for AWS") } awsConfig += fmt.Sprintf("Zone=%s\n", cloudConfig.Zone) if cloudConfig.ClusterTag == "" { return fmt.Errorf("--cluster-tag must be specified for AWS") } awsConfig += fmt.Sprintf("KubernetesClusterTag=%s\n", cloudConfig.ClusterTag) var err error cloudConfig.Provider, err = cloudprovider.GetCloudProvider(testContext.Provider, strings.NewReader(awsConfig)) if err != nil { return fmt.Errorf("Error building AWS provider: %v", err) } } return nil }
// getGCECloudProvider returns the GCE cloud provider, for use for querying volume labels func (l *persistentVolumeLabel) getGCECloudProvider() (*gce.GCECloud, error) { l.mutex.Lock() defer l.mutex.Unlock() if l.gceCloudProvider == nil { cloudProvider, err := cloudprovider.GetCloudProvider("gce", nil) if err != nil || cloudProvider == nil { return nil, err } gceCloudProvider, ok := cloudProvider.(*gce.GCECloud) if !ok { // GetCloudProvider has gone very wrong return nil, fmt.Errorf("error retrieving GCE cloud provider") } l.gceCloudProvider = gceCloudProvider } return l.gceCloudProvider, nil }
// getEBSVolumes returns the AWS Volumes interface for ebs func (l *persistentVolumeLabel) getEBSVolumes() (aws.Volumes, error) { l.mutex.Lock() defer l.mutex.Unlock() if l.ebsVolumes == nil { cloudProvider, err := cloudprovider.GetCloudProvider("aws", nil) if err != nil || cloudProvider == nil { return nil, err } awsCloudProvider, ok := cloudProvider.(*aws.AWSCloud) if !ok { // GetCloudProvider has gone very wrong return nil, fmt.Errorf("error retrieving AWS cloud provider") } l.ebsVolumes = awsCloudProvider } return l.ebsVolumes, nil }
// NewClusterManager creates a cluster manager for shared resources. // - name: is the name used to tag cluster wide shared resources. This is the // string passed to glbc via --gce-cluster-name. // - defaultBackendNodePort: is the node port of glbc's default backend. This is // the kubernetes Service that serves the 404 page if no urls match. // - defaultHealthCheckPath: is the default path used for L7 health checks, eg: "/healthz" func NewClusterManager( name string, defaultBackendNodePort int64, defaultHealthCheckPath string) (*ClusterManager, error) { cloudInterface, err := cloudprovider.GetCloudProvider("gce", nil) if err != nil { return nil, err } cloud := cloudInterface.(*gce.GCECloud) cluster := ClusterManager{ClusterName: name} cluster.instancePool = NewNodePool(cloud) healthChecker := NewHealthChecker(cloud, defaultHealthCheckPath) cluster.backendPool = NewBackendPool( cloud, healthChecker, cluster.instancePool) defaultBackendHealthChecker := NewHealthChecker(cloud, "/healthz") defaultBackendPool := NewBackendPool( cloud, defaultBackendHealthChecker, cluster.instancePool) cluster.defaultBackendNodePort = defaultBackendNodePort cluster.l7Pool = NewLoadBalancerPool( cloud, defaultBackendPool, defaultBackendNodePort) return &cluster, nil }
// Enabled implements DockerConfigProvider.Enabled for the AWS token-based implementation. // For now, it gets activated only if AWS was chosen as the cloud provider. // TODO: figure how to enable it manually for deployments that are not on AWS but still // use ECR somehow? func (p *ecrProvider) Enabled() bool { provider, err := cloudprovider.GetCloudProvider(aws_cloud.ProviderName, nil) if err != nil { glog.Errorf("while initializing AWS cloud provider %v", err) return false } if provider == nil { return false } zones, ok := provider.Zones() if !ok { glog.Errorf("couldn't get Zones() interface") return false } zone, err := zones.GetZone() if err != nil { glog.Errorf("while getting zone %v", err) return false } if zone.Region == "" { glog.Errorf("Region information is empty") return false } getter := &ecrTokenGetter{svc: ecr.New(session.New(&aws.Config{ Credentials: nil, Region: &zone.Region, }))} getter.svc.Handlers.Sign.PushFrontNamed(request.NamedHandler{ Name: "k8s/logger", Fn: awsHandlerLogger, }) p.getter = getter return true }
// Detaches the specified persistent disk device from node, verifies that it is detached, and retries if it fails. // This function is intended to be called asynchronously as a go routine. // It starts the detachCleanupManager with the specified pdName so that callers can wait for completion. func detachDiskAndVerify(c *gcePersistentDiskCleaner) { glog.V(5).Infof("detachDiskAndVerify for pd %q.", c.pdName) defer util.HandleCrash() // Start operation, so that other threads can wait on this detach operation. // Set bufferSize to 0 so senders are blocked on send until we receive. ch, err := detachCleanupManager.Start(c.pdName, 0 /* bufferSize */) if err != nil { glog.Errorf("Error adding %q to detachCleanupManager: %v", c.pdName, err) return } defer detachCleanupManager.Close(c.pdName) defer func() { // Unblock any callers that have been waiting for this detach routine to complete. for { select { case <-ch: glog.V(5).Infof("detachDiskAndVerify for pd %q clearing chan.", c.pdName) default: glog.V(5).Infof("detachDiskAndVerify for pd %q done clearing chans.", c.pdName) return } } }() devicePaths := getDiskByIdPaths(c.gcePersistentDisk) var gce cloudprovider.Interface for numRetries := 0; numRetries < maxRetries; numRetries++ { if gce == nil { var err error gce, err = cloudprovider.GetCloudProvider("gce", nil) if err != nil || gce == nil { // Retry on error. See issue #11321 glog.Errorf("Error getting GCECloudProvider while detaching PD %q: %v", c.pdName, err) gce = nil time.Sleep(errorSleepDuration) continue } } if numRetries > 0 { glog.Warningf("Timed out waiting for GCE PD %q to detach. Retrying detach.", c.pdName) } if err := gce.(*gce_cloud.GCECloud).DetachDisk(c.pdName); err != nil { // Retry on error. See issue #11321. Continue and verify if disk is detached, because a // previous detach operation may still succeed. glog.Errorf("Error detaching PD %q: %v", c.pdName, err) } for numChecks := 0; numChecks < maxChecks; numChecks++ { allPathsRemoved := true for _, path := range devicePaths { if err := udevadmChangeToDrive(path); err != nil { // udevadm errors should not block disk detachment, log and continue glog.Errorf("%v", err) } if exists, err := pathExists(path); err != nil { // Retry on error. See issue #11321 glog.Errorf("Error checking if path exists: %v", err) } else { allPathsRemoved = allPathsRemoved && !exists } } if allPathsRemoved { // All paths to the PD have been succefully removed glog.Infof("Successfully detached GCE PD %q.", c.pdName) return } // Sleep then check again glog.V(3).Infof("Waiting for GCE PD %q to detach.", c.pdName) time.Sleep(checkSleepDuration) } } glog.Errorf("Failed to detach GCE PD %q. One or more mount paths was not removed.", c.pdName) }
func TestE2E(t *testing.T) { util.ReallyCrash = true util.InitLogs() defer util.FlushLogs() if *reportDir != "" { if err := os.MkdirAll(*reportDir, 0755); err != nil { glog.Errorf("Failed creating report directory: %v", err) } defer CoreDump(*reportDir) } if testContext.Provider == "" { glog.Info("The --provider flag is not set. Treating as a conformance test. Some tests may not be run.") } if testContext.Provider == "gce" || testContext.Provider == "gke" { var err error Logf("Fetching cloud provider for %q\r\n", testContext.Provider) var tokenSource oauth2.TokenSource tokenSource = nil if cloudConfig.ServiceAccount != "" { // Use specified service account for auth Logf("Using service account %q as token source.", cloudConfig.ServiceAccount) tokenSource = google.ComputeTokenSource(cloudConfig.ServiceAccount) } cloudConfig.Provider, err = gcecloud.CreateGCECloud(testContext.CloudConfig.ProjectID, testContext.CloudConfig.Zone, "" /* networkUrl */, tokenSource, false /* useMetadataServer */) if err != nil { glog.Fatal("Error building GCE provider: ", err) } } if testContext.Provider == "aws" { awsConfig := "[Global]\n" if cloudConfig.Zone == "" { glog.Fatal("gce-zone must be specified for AWS") } awsConfig += fmt.Sprintf("Zone=%s\n", cloudConfig.Zone) if cloudConfig.ClusterTag == "" { glog.Fatal("--cluster-tag must be specified for AWS") } awsConfig += fmt.Sprintf("KubernetesClusterTag=%s\n", cloudConfig.ClusterTag) var err error cloudConfig.Provider, err = cloudprovider.GetCloudProvider(testContext.Provider, strings.NewReader(awsConfig)) if err != nil { glog.Fatal("Error building AWS provider: ", err) } } // Disable skipped tests unless they are explicitly requested. if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" { config.GinkgoConfig.SkipString = `\[Skipped\]` } gomega.RegisterFailHandler(ginkgo.Fail) c, err := loadClient() if err != nil { glog.Fatal("Error loading client: ", err) } // Delete any namespaces except default and kube-system. This ensures no // lingering resources are left over from a previous test run. if testContext.CleanStart { deleted, err := deleteNamespaces(c, nil /* deleteFilter */, []string{api.NamespaceSystem, api.NamespaceDefault}) if err != nil { t.Errorf("Error deleting orphaned namespaces: %v", err) } glog.Infof("Waiting for deletion of the following namespaces: %v", deleted) if err := waitForNamespacesDeleted(c, deleted, namespaceCleanupTimeout); err != nil { glog.Fatalf("Failed to delete orphaned namespaces %v: %v", deleted, err) } } // Ensure all pods are running and ready before starting tests (otherwise, // cluster infrastructure pods that are being pulled or started can block // test pods from running, and tests that ensure all pods are running and // ready will fail). if err := waitForPodsRunningReady(api.NamespaceSystem, testContext.MinStartupPods, podStartupTimeout); err != nil { t.Errorf("Error waiting for all pods to be running and ready: %v", err) return } // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins var r []ginkgo.Reporter if *reportDir != "" { r = append(r, reporters.NewJUnitReporter(path.Join(*reportDir, fmt.Sprintf("junit_%02d.xml", config.GinkgoConfig.ParallelNode)))) } glog.Infof("Starting e2e run; %q", runId) ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes e2e suite", r) }