func TestOVirtCloudConfiguration(t *testing.T) {
config1 := (io.Reader)(nil)
_, err1 := cloudprovider.GetCloudProvider("ovirt", config1)
if err1 == nil {
t.Fatalf("An error is expected when the configuration is missing")
}
config2 := strings.NewReader("")
_, err2 := cloudprovider.GetCloudProvider("ovirt", config2)
if err2 == nil {
t.Fatalf("An error is expected when the configuration is empty")
}
config3 := strings.NewReader(`
[connection]
`)
_, err3 := cloudprovider.GetCloudProvider("ovirt", config3)
if err3 == nil {
t.Fatalf("An error is expected when the uri is missing")
}
config4 := strings.NewReader(`
[connection]
uri = https://localhost:8443/ovirt-engine/api
`)
_, err4 := cloudprovider.GetCloudProvider("ovirt", config4)
if err4 != nil {
t.Fatalf("Unexpected error creating the provider: %s", err4)
}
}
// Unmounts the device and detaches the disk from the kubelet's host machine.
// Expects a GCE device path symlink. Ex: /dev/disk/by-id/google-mydisk-part1
func (util *GCEDiskUtil) DetachDisk(pd *gcePersistentDisk, devicePath string) error {
// Follow the symlink to the actual device path.
canonicalDevicePath, err := filepath.EvalSymlinks(devicePath)
if err != nil {
return err
}
deviceName, err := getDeviceName(devicePath, canonicalDevicePath)
if err != nil {
return err
}
globalPDPath := makeGlobalPDName(pd.plugin.host, deviceName, pd.readOnly)
if err := pd.mounter.Unmount(globalPDPath, 0); err != nil {
return err
}
if err := os.RemoveAll(globalPDPath); err != nil {
return err
}
gce, err := cloudprovider.GetCloudProvider("gce", nil)
if err != nil {
return err
}
if err := gce.(*gce_cloud.GCECloud).DetachDisk(deviceName); err != nil {
return err
}
return nil
}
// Unmounts the device and detaches the disk from the kubelet's host machine.
func (util *GCEDiskUtil) DetachDisk(pd *gcePersistentDisk) error {
// Unmount the global PD mount, which should be the only one.
globalPDPath := makeGlobalPDName(pd.plugin.host, pd.pdName)
glog.V(5).Infof("DetachDisk(pd) where pd is %#v and the globalPDPath is %q\r\n", pd, globalPDPath)
// Terminate any in progress verify detach go routines, this will block until the goroutine is ready to exit because the channel is unbuffered
detachCleanupManager.Send(pd.pdName, true)
if err := pd.mounter.Unmount(globalPDPath); err != nil {
return err
}
if err := os.Remove(globalPDPath); err != nil {
return err
}
// Detach the disk
gce, err := cloudprovider.GetCloudProvider("gce", nil)
if err != nil {
return err
}
if err := gce.(*gce_cloud.GCECloud).DetachDisk(pd.pdName); err != nil {
return err
}
// Verify disk detached, retry if needed.
go verifyDetached(pd, gce)
return nil
}
func initCloudProvider(name string, configFilePath string) cloudprovider.Interface {
var config *os.File
if name == "" {
glog.Info("No cloud provider specified.")
return nil
}
if configFilePath != "" {
var err error
config, err = os.Open(configFilePath)
if err != nil {
glog.Fatalf("Couldn't open cloud provider configuration %s: %#v",
configFilePath, err)
}
defer config.Close()
}
cloud, err := cloudprovider.GetCloudProvider(name, config)
if err != nil {
glog.Fatalf("Couldn't init cloud provider %q: %#v", name, err)
}
if cloud == nil {
glog.Fatalf("Unknown cloud provider: %s", name)
}
return cloud
}
func main() {
util.InitFlags()
goruntime.GOMAXPROCS(goruntime.NumCPU())
if context.Provider == "" {
glog.Info("The --provider flag is not set. Treating as a conformance test. Some tests may not be run.")
os.Exit(1)
}
if *times <= 0 {
glog.Error("Invalid --times (negative or no testing requested)!")
os.Exit(1)
}
if context.Provider == "aws" {
awsConfig := "[Global]\n"
if cloudConfig.Zone == "" {
glog.Error("gce_zone must be specified for AWS")
os.Exit(1)
}
awsConfig += fmt.Sprintf("Zone=%s\n", cloudConfig.Zone)
var err error
cloudConfig.Provider, err = cloudprovider.GetCloudProvider(context.Provider, strings.NewReader(awsConfig))
if err != nil {
glog.Error("Error building AWS provider: ", err)
os.Exit(1)
}
}
e2e.RunE2ETests(context, *orderseed, *times, *reportDir, testList)
}
func TestE2E(t *testing.T) {
util.ReallyCrash = true
util.InitLogs()
defer util.FlushLogs()
// TODO: possibly clean up or refactor this functionality.
if testContext.Provider == "" {
glog.Fatal("The --provider flag is not set. Treating as a conformance test. Some tests may not be run.")
}
if testContext.Provider == "aws" {
awsConfig := "[Global]\n"
if cloudConfig.Zone == "" {
glog.Fatal("gce-zone must be specified for AWS")
}
awsConfig += fmt.Sprintf("Zone=%s\n", cloudConfig.Zone)
if cloudConfig.ClusterTag == "" {
glog.Fatal("--cluster-tag must be specified for AWS")
}
awsConfig += fmt.Sprintf("KubernetesClusterTag=%s\n", cloudConfig.ClusterTag)
var err error
cloudConfig.Provider, err = cloudprovider.GetCloudProvider(testContext.Provider, strings.NewReader(awsConfig))
if err != nil {
glog.Fatal("Error building AWS provider: ", err)
}
}
// Disable density test unless it's explicitly requested.
if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" {
config.GinkgoConfig.SkipString = "Skipped"
}
gomega.RegisterFailHandler(ginkgo.Fail)
// Ensure all pods are running and ready before starting tests (otherwise,
// cluster infrastructure pods that are being pulled or started can block
// test pods from running, and tests that ensure all pods are running and
// ready will fail).
if err := waitForPodsRunningReady(api.NamespaceDefault, testContext.MinStartupPods, podStartupTimeout); err != nil {
glog.Fatalf("Error waiting for all pods to be running and ready: %v", err)
}
// Run tests through the Ginkgo runner with output to console + JUnit for Jenkins
var r []ginkgo.Reporter
if *reportDir != "" {
r = append(r, reporters.NewJUnitReporter(path.Join(*reportDir, fmt.Sprintf("junit_%02d.xml", config.GinkgoConfig.ParallelNode))))
failReport := &failReporter{}
r = append(r, failReport)
defer func() {
if failReport.failed {
coreDump(*reportDir)
}
}()
}
ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes e2e suite", r)
}
func main() {
flag.Parse()
util.InitLogs()
defer util.FlushLogs()
verflag.PrintAndExitIfRequested()
verifyMinionFlags()
if len(etcdServerList) == 0 {
glog.Fatalf("-etcd_servers flag is required.")
}
cloud, err := cloudprovider.GetCloudProvider(*cloudProvider)
if err != nil {
glog.Fatalf("Couldn't init cloud provider %q: %#v", *cloudProvider, err)
}
if cloud == nil {
if len(*cloudProvider) > 0 {
glog.Fatalf("Unknown cloud provider: %s", *cloudProvider)
} else {
glog.Info("No cloud provider specified.")
}
}
podInfoGetter := &client.HTTPPodInfoGetter{
Client: http.DefaultClient,
Port: *minionPort,
}
client, err := client.New(net.JoinHostPort(*address, strconv.Itoa(int(*port))), nil)
if err != nil {
glog.Fatalf("Invalid server address: %v", err)
}
m := master.New(&master.Config{
Client: client,
Cloud: cloud,
EtcdServers: etcdServerList,
HealthCheckMinions: *healthCheckMinions,
Minions: machineList,
MinionCacheTTL: *minionCacheTTL,
MinionRegexp: *minionRegexp,
PodInfoGetter: podInfoGetter,
})
storage, codec := m.API_v1beta1()
s := &http.Server{
Addr: net.JoinHostPort(*address, strconv.Itoa(int(*port))),
Handler: apiserver.Handle(storage, codec, *apiPrefix),
ReadTimeout: 5 * time.Minute,
WriteTimeout: 5 * time.Minute,
MaxHeaderBytes: 1 << 20,
}
glog.Fatal(s.ListenAndServe())
}
// Attaches a disk specified by a volume.GCEPersistentDisk to the current kubelet.
// Mounts the disk to it's global path.
func (util *GCEDiskUtil) AttachDisk(pd *gcePersistentDisk) error {
gce, err := cloudprovider.GetCloudProvider("gce", nil)
if err != nil {
return err
}
flags := uintptr(0)
if pd.readOnly {
flags = mount.FlagReadOnly
}
if err := gce.(*gce_cloud.GCECloud).AttachDisk(pd.pdName, pd.readOnly); err != nil {
return err
}
devicePath := path.Join("/dev/disk/by-id/", "google-"+pd.pdName)
if pd.partition != "" {
devicePath = devicePath + "-part" + pd.partition
}
//TODO(jonesdl) There should probably be better method than busy-waiting here.
numTries := 0
for {
_, err := os.Stat(devicePath)
if err == nil {
break
}
if err != nil && !os.IsNotExist(err) {
return err
}
numTries++
if numTries == 10 {
return errors.New("Could not attach disk: Timeout after 10s")
}
time.Sleep(time.Second)
}
globalPDPath := makeGlobalPDName(pd.plugin.host, pd.pdName, pd.readOnly)
// Only mount the PD globally once.
mountpoint, err := isMountPoint(globalPDPath)
if err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(globalPDPath, 0750); err != nil {
return err
}
mountpoint = false
} else {
return err
}
}
if !mountpoint {
err = pd.mounter.Mount(devicePath, globalPDPath, pd.fsType, flags, "")
if err != nil {
os.RemoveAll(globalPDPath)
return err
}
}
return nil
}
// getVolumeProvider returns the AWS Volumes interface
func (ebs *awsElasticBlockStore) getVolumeProvider() (aws_cloud.Volumes, error) {
name := "aws"
cloud, err := cloudprovider.GetCloudProvider(name, nil)
if err != nil {
return nil, err
}
volumes, ok := cloud.(aws_cloud.Volumes)
if !ok {
return nil, fmt.Errorf("Cloud provider does not support volumes")
}
return volumes, nil
}
// Attaches the specified persistent disk device to node, verifies that it is attached, and retries if it fails.
func attachDiskAndVerify(pd *gcePersistentDisk, sdBeforeSet util.StringSet) (string, error) {
devicePaths := getDiskByIdPaths(pd)
var gce cloudprovider.Interface
for numRetries := 0; numRetries < maxRetries; numRetries++ {
if gce == nil {
var err error
gce, err = cloudprovider.GetCloudProvider("gce", nil)
if err != nil || gce == nil {
// Retry on error. See issue #11321
glog.Errorf("Error getting GCECloudProvider while attaching PD %q: %v", pd.pdName, err)
gce = nil
time.Sleep(errorSleepDuration)
continue
}
}
if numRetries > 0 {
glog.Warningf("Timed out waiting for GCE PD %q to attach. Retrying attach.", pd.pdName)
}
if err := gce.(*gce_cloud.GCECloud).AttachDisk(pd.pdName, pd.readOnly); err != nil {
// Retry on error. See issue #11321. Continue and verify if disk is attached, because a
// previous attach operation may still succeed.
glog.Errorf("Error attaching PD %q: %v", pd.pdName, err)
}
for numChecks := 0; numChecks < maxChecks; numChecks++ {
if err := udevadmChangeToNewDrives(sdBeforeSet); err != nil {
// udevadm errors should not block disk attachment, log and continue
glog.Errorf("%v", err)
}
for _, path := range devicePaths {
if pathExists, err := pathExists(path); err != nil {
// Retry on error. See issue #11321
glog.Errorf("Error checking if path exists: %v", err)
} else if pathExists {
// A device path has succesfully been created for the PD
glog.Infof("Succesfully attached GCE PD %q.", pd.pdName)
return path, nil
}
}
// Sleep then check again
glog.V(3).Infof("Waiting for GCE PD %q to attach.", pd.pdName)
time.Sleep(checkSleepDuration)
}
}
return "", fmt.Errorf("Could not attach GCE PD %q. Timeout waiting for mount paths to be created.", pd.pdName)
}
// Attaches a disk specified by a volume.GCEPersistentDisk to the current kubelet.
// Mounts the disk to it's global path.
func (util *GCEDiskUtil) AttachDisk(GCEPD *GCEPersistentDisk) error {
gce, err := cloudprovider.GetCloudProvider("gce", nil)
if err != nil {
return err
}
flags := uintptr(0)
if GCEPD.ReadOnly {
flags = MOUNT_MS_RDONLY
}
if err := gce.(*gce_cloud.GCECloud).AttachDisk(GCEPD.PDName, GCEPD.ReadOnly); err != nil {
return err
}
devicePath := path.Join("/dev/disk/by-id/", "google-"+GCEPD.PDName)
if GCEPD.Partition != "" {
devicePath = devicePath + "-part" + GCEPD.Partition
}
//TODO(jonesdl) There should probably be better method than busy-waiting here.
numTries := 0
for {
_, err := os.Stat(devicePath)
if err == nil {
break
}
if err != nil && !os.IsNotExist(err) {
return err
}
numTries++
if numTries == 10 {
return errors.New("Could not attach disk: Timeout after 10s")
}
time.Sleep(time.Second)
}
globalPDPath := makeGlobalPDName(GCEPD.RootDir, GCEPD.PDName, GCEPD.ReadOnly)
// Only mount the PD globally once.
_, err = os.Stat(globalPDPath)
if os.IsNotExist(err) {
err = os.MkdirAll(globalPDPath, 0750)
if err != nil {
return err
}
err = GCEPD.mounter.Mount(devicePath, globalPDPath, GCEPD.FSType, flags, "")
if err != nil {
os.RemoveAll(globalPDPath)
return err
}
} else if err != nil {
return err
}
return nil
}
// Attaches a disk specified by a volume.GCEPersistentDisk to the current kubelet.
// Mounts the disk to it's global path.
func (diskUtil *GCEDiskUtil) AttachAndMountDisk(pd *gcePersistentDisk, globalPDPath string) error {
glog.V(5).Infof("AttachAndMountDisk(pd, %q) where pd is %#v\r\n", globalPDPath, pd)
// Terminate any in progress verify detach go routines, this will block until the goroutine is ready to exit because the channel is unbuffered
detachCleanupManager.Send(pd.pdName, true)
sdBefore, err := filepath.Glob(diskSDPattern)
if err != nil {
glog.Errorf("Error filepath.Glob(\"%s\"): %v\r\n", diskSDPattern, err)
}
sdBeforeSet := util.NewStringSet(sdBefore...)
gce, err := cloudprovider.GetCloudProvider("gce", nil)
if err != nil {
return err
}
if err := gce.(*gce_cloud.GCECloud).AttachDisk(pd.pdName, pd.readOnly); err != nil {
return err
}
devicePath, err := verifyAttached(pd, sdBeforeSet, gce)
if err != nil {
return err
}
// Only mount the PD globally once.
mountpoint, err := pd.mounter.IsMountPoint(globalPDPath)
if err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(globalPDPath, 0750); err != nil {
return err
}
mountpoint = false
} else {
return err
}
}
options := []string{}
if pd.readOnly {
options = append(options, "ro")
}
if !mountpoint {
err = pd.diskMounter.Mount(devicePath, globalPDPath, pd.fsType, options)
if err != nil {
os.Remove(globalPDPath)
return err
}
}
return nil
}
// Unmounts the device and detaches the disk from the kubelet's host machine.
func (util *GCEDiskUtil) DetachDisk(pd *gcePersistentDisk) error {
// Unmount the global PD mount, which should be the only one.
globalPDPath := makeGlobalPDName(pd.plugin.host, pd.pdName)
if err := pd.mounter.Unmount(globalPDPath); err != nil {
return err
}
if err := os.Remove(globalPDPath); err != nil {
return err
}
// Detach the disk
gce, err := cloudprovider.GetCloudProvider("gce", nil)
if err != nil {
return err
}
if err := gce.(*gce_cloud.GCECloud).DetachDisk(pd.pdName); err != nil {
return err
}
return nil
}
// Attaches a disk specified by a volume.GCEPersistentDisk to the current kubelet.
// Mounts the disk to it's global path.
func (util *GCEDiskUtil) AttachAndMountDisk(pd *gcePersistentDisk, globalPDPath string) error {
gce, err := cloudprovider.GetCloudProvider("gce", nil)
if err != nil {
return err
}
if err := gce.(*gce_cloud.GCECloud).AttachDisk(pd.pdName, pd.readOnly); err != nil {
return err
}
devicePaths := []string{
path.Join("/dev/disk/by-id/", "google-"+pd.pdName),
path.Join("/dev/disk/by-id/", "scsi-0Google_PersistentDisk_"+pd.pdName),
}
if pd.partition != "" {
for i, path := range devicePaths {
devicePaths[i] = path + "-part" + pd.partition
}
}
//TODO(jonesdl) There should probably be better method than busy-waiting here.
numTries := 0
devicePath := ""
// Wait for the disk device to be created
for {
for _, path := range devicePaths {
_, err := os.Stat(path)
if err == nil {
devicePath = path
break
}
if err != nil && !os.IsNotExist(err) {
return err
}
}
if devicePath != "" {
break
}
numTries++
if numTries == 10 {
return errors.New("Could not attach disk: Timeout after 10s")
}
time.Sleep(time.Second)
}
// Only mount the PD globally once.
mountpoint, err := pd.mounter.IsMountPoint(globalPDPath)
if err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(globalPDPath, 0750); err != nil {
return err
}
mountpoint = false
} else {
return err
}
}
options := []string{}
if pd.readOnly {
options = append(options, "ro")
}
if !mountpoint {
err = pd.diskMounter.Mount(devicePath, globalPDPath, pd.fsType, options)
if err != nil {
os.Remove(globalPDPath)
return err
}
}
return nil
}
// Detaches the specified persistent disk device from node, verifies that it is detached, and retries if it fails.
// This function is intended to be called asynchronously as a go routine.
// It starts the detachCleanupManager with the specified pdName so that callers can wait for completion.
func detachDiskAndVerify(pd *gcePersistentDisk) {
glog.V(5).Infof("detachDiskAndVerify for pd %q.", pd.pdName)
defer util.HandleCrash()
// Start operation, so that other threads can wait on this detach operation.
// Set bufferSize to 0 so senders are blocked on send until we recieve.
ch, err := detachCleanupManager.Start(pd.pdName, 0 /* bufferSize */)
if err != nil {
glog.Errorf("Error adding %q to detachCleanupManager: %v", pd.pdName, err)
return
}
defer detachCleanupManager.Close(pd.pdName)
defer func() {
// Unblock any callers that have been waiting for this detach routine to complete.
for {
select {
case <-ch:
glog.V(5).Infof("detachDiskAndVerify for pd %q clearing chan.", pd.pdName)
default:
glog.V(5).Infof("detachDiskAndVerify for pd %q done clearing chans.", pd.pdName)
return
}
}
}()
devicePaths := getDiskByIdPaths(pd)
var gce cloudprovider.Interface
for numRetries := 0; numRetries < maxRetries; numRetries++ {
if gce == nil {
var err error
gce, err = cloudprovider.GetCloudProvider("gce", nil)
if err != nil || gce == nil {
// Retry on error. See issue #11321
glog.Errorf("Error getting GCECloudProvider while detaching PD %q: %v", pd.pdName, err)
gce = nil
time.Sleep(errorSleepDuration)
continue
}
}
if numRetries > 0 {
glog.Warningf("Timed out waiting for GCE PD %q to detach. Retrying detach.", pd.pdName)
}
if err := gce.(*gce_cloud.GCECloud).DetachDisk(pd.pdName); err != nil {
// Retry on error. See issue #11321. Continue and verify if disk is detached, because a
// previous detach operation may still succeed.
glog.Errorf("Error detaching PD %q: %v", pd.pdName, err)
}
for numChecks := 0; numChecks < maxChecks; numChecks++ {
allPathsRemoved := true
for _, path := range devicePaths {
if err := udevadmChangeToDrive(path); err != nil {
// udevadm errors should not block disk detachment, log and continue
glog.Errorf("%v", err)
}
if exists, err := pathExists(path); err != nil {
// Retry on error. See issue #11321
glog.Errorf("Error checking if path exists: %v", err)
} else {
allPathsRemoved = allPathsRemoved && !exists
}
}
if allPathsRemoved {
// All paths to the PD have been succefully removed
glog.Infof("Succesfully detached GCE PD %q.", pd.pdName)
return
}
// Sleep then check again
glog.V(3).Infof("Waiting for GCE PD %q to detach.", pd.pdName)
time.Sleep(checkSleepDuration)
}
}
glog.Errorf("Failed to detach GCE PD %q. One or more mount paths was not removed.", pd.pdName)
}