func TestDeleter(t *testing.T) {
tempPath := fmt.Sprintf("/tmp/hostpath/%s", util.NewUUID())
defer os.RemoveAll(tempPath)
err := os.MkdirAll(tempPath, 0750)
if err != nil {
t.Fatalf("Failed to create tmp directory for deleter: %v", err)
}
plugMgr := volume.VolumePluginMgr{}
plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), volume.NewFakeVolumeHost("/tmp/fake", nil, nil))
spec := &volume.Spec{PersistentVolume: &api.PersistentVolume{Spec: api.PersistentVolumeSpec{PersistentVolumeSource: api.PersistentVolumeSource{HostPath: &api.HostPathVolumeSource{Path: tempPath}}}}}
plug, err := plugMgr.FindDeletablePluginBySpec(spec)
if err != nil {
t.Errorf("Can't find the plugin by name")
}
deleter, err := plug.NewDeleter(spec)
if err != nil {
t.Errorf("Failed to make a new Deleter: %v", err)
}
if deleter.GetPath() != tempPath {
t.Errorf("Expected %s but got %s", tempPath, deleter.GetPath())
}
if err := deleter.Delete(); err != nil {
t.Errorf("Mock Recycler expected to return nil but got %s", err)
}
if exists, _ := util.FileExists("foo"); exists {
t.Errorf("Temp path expected to be deleted, but was found at %s", tempPath)
}
}
func (m *Master) generateSSHKey(user, privateKeyfile, publicKeyfile string) error {
private, public, err := util.GenerateKey(2048)
if err != nil {
return err
}
// If private keyfile already exists, we must have only made it halfway
// through last time, so delete it.
exists, err := util.FileExists(privateKeyfile)
if err != nil {
glog.Errorf("Error detecting if private key exists: %v", err)
} else if exists {
glog.Infof("Private key exists, but public key does not")
if err := os.Remove(privateKeyfile); err != nil {
glog.Errorf("Failed to remove stale private key: %v", err)
}
}
if err := ioutil.WriteFile(privateKeyfile, util.EncodePrivateKey(private), 0600); err != nil {
return err
}
publicKeyBytes, err := util.EncodePublicKey(public)
if err != nil {
return err
}
if err := ioutil.WriteFile(publicKeyfile+".tmp", publicKeyBytes, 0600); err != nil {
return err
}
return os.Rename(publicKeyfile+".tmp", publicKeyfile)
}
func getAppArmorFS() (string, error) {
mountsFile, err := os.Open("/proc/mounts")
if err != nil {
return "", fmt.Errorf("could not open /proc/mounts: %v", err)
}
defer mountsFile.Close()
scanner := bufio.NewScanner(mountsFile)
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if len(fields) < 3 {
// Unknown line format; skip it.
continue
}
if fields[2] == "securityfs" {
appArmorFS := path.Join(fields[1], "apparmor")
if ok, err := util.FileExists(appArmorFS); !ok {
msg := fmt.Sprintf("path %s does not exist", appArmorFS)
if err != nil {
return "", fmt.Errorf("%s: %v", msg, err)
} else {
return "", errors.New(msg)
}
} else {
return appArmorFS, nil
}
}
}
if err := scanner.Err(); err != nil {
return "", fmt.Errorf("error scanning mounts: %v", err)
}
return "", errors.New("securityfs not found")
}
// Run establishes tunnel loops and returns
func (c *SSHTunneler) Run(getAddresses AddressFunc) {
if c.stopChan != nil {
return
}
c.stopChan = make(chan struct{})
// Save the address getter
if getAddresses != nil {
c.getAddresses = getAddresses
}
// Usernames are capped @ 32
if len(c.SSHUser) > 32 {
glog.Warning("SSH User is too long, truncating to 32 chars")
c.SSHUser = c.SSHUser[0:32]
}
glog.Infof("Setting up proxy: %s %s", c.SSHUser, c.SSHKeyfile)
// public keyfile is written last, so check for that.
publicKeyFile := c.SSHKeyfile + ".pub"
exists, err := util.FileExists(publicKeyFile)
if err != nil {
glog.Errorf("Error detecting if key exists: %v", err)
} else if !exists {
glog.Infof("Key doesn't exist, attempting to create")
err := c.generateSSHKey(c.SSHUser, c.SSHKeyfile, publicKeyFile)
if err != nil {
glog.Errorf("Failed to create key pair: %v", err)
}
}
c.tunnels = &util.SSHTunnelList{}
c.setupSecureProxy(c.SSHUser, c.SSHKeyfile, publicKeyFile)
c.lastSync = c.clock.Now().Unix()
}
// checks if the required cgroups subsystems are mounted.
// As of now, only 'cpu' and 'memory' are required.
// cpu quota is a soft requirement.
func validateSystemRequirements(mountUtil mount.Interface) (features, error) {
const (
cgroupMountType = "cgroup"
localErr = "system validation failed"
)
var (
cpuMountPoint string
f features
)
mountPoints, err := mountUtil.List()
if err != nil {
return f, fmt.Errorf("%s - %v", localErr, err)
}
expectedCgroups := sets.NewString("cpu", "cpuacct", "cpuset", "memory")
for _, mountPoint := range mountPoints {
if mountPoint.Type == cgroupMountType {
for _, opt := range mountPoint.Opts {
if expectedCgroups.Has(opt) {
expectedCgroups.Delete(opt)
}
if opt == "cpu" {
cpuMountPoint = mountPoint.Path
}
}
}
}
if expectedCgroups.Len() > 0 {
return f, fmt.Errorf("%s - Following Cgroup subsystem not mounted: %v", localErr, expectedCgroups.List())
}
// Check if cpu quota is available.
// CPU cgroup is required and so it expected to be mounted at this point.
periodExists, err := util.FileExists(path.Join(cpuMountPoint, "cpu.cfs_period_us"))
if err != nil {
glog.Errorf("failed to detect if CPU cgroup cpu.cfs_period_us is available - %v", err)
}
quotaExists, err := util.FileExists(path.Join(cpuMountPoint, "cpu.cfs_quota_us"))
if err != nil {
glog.Errorf("failed to detect if CPU cgroup cpu.cfs_quota_us is available - %v", err)
}
if quotaExists && periodExists {
f.cpuHardcapping = true
}
return f, nil
}
// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (c *cinderVolumeCleaner) TearDownAt(dir string) error {
notmnt, err := c.mounter.IsLikelyNotMountPoint(dir)
if err != nil {
return err
}
exist, _ := util.FileExists(path.Join(dir, OpenStackCloudProviderTagFile))
if exist {
c.withOpenStackCP = false
} else {
c.withOpenStackCP = true
}
if notmnt {
if !c.withOpenStackCP && c.isNoMountSupported {
volumeID, err := ioutil.ReadFile(path.Join(dir, OpenStackCloudProviderTagFile))
if err != nil {
return err
}
c.pdName = string(volumeID)
if err := c.manager.DetachDisk(c); err != nil {
return err
}
}
return os.RemoveAll(dir)
}
refs, err := mount.GetMountRefs(c.mounter, dir)
if err != nil {
return err
}
if err := c.mounter.Unmount(dir); err != nil {
return err
}
glog.Infof("successfully unmounted: %s\n", dir)
// If refCount is 1, then all bind mounts have been removed, and the
// remaining reference is the global mount. It is safe to detach.
if len(refs) == 1 {
c.pdName = path.Base(refs[0])
if err := c.manager.DetachDisk(c); err != nil {
return err
}
}
notmnt, mntErr := c.mounter.IsLikelyNotMountPoint(dir)
if mntErr != nil {
glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
return err
}
if !notmnt {
if err := os.Remove(dir); err != nil {
return err
}
}
return nil
}
// init initializes master.
func (m *Master) init(c *Config) {
healthzChecks := []healthz.HealthzChecker{}
m.clock = util.RealClock{}
podStorage := podetcd.NewStorage(c.DatabaseStorage, c.KubeletClient)
podTemplateStorage := podtemplateetcd.NewREST(c.DatabaseStorage)
eventRegistry := event.NewEtcdRegistry(c.DatabaseStorage, uint64(c.EventTTL.Seconds()))
limitRangeStorage := limitrangeetcd.NewStorage(c.DatabaseStorage)
resourceQuotaStorage, resourceQuotaStatusStorage := resourcequotaetcd.NewStorage(c.DatabaseStorage)
secretStorage := secretetcd.NewStorage(c.DatabaseStorage)
serviceAccountStorage := serviceaccountetcd.NewStorage(c.DatabaseStorage)
persistentVolumeStorage, persistentVolumeStatusStorage := pvetcd.NewStorage(c.DatabaseStorage)
persistentVolumeClaimStorage, persistentVolumeClaimStatusStorage := pvcetcd.NewStorage(c.DatabaseStorage)
namespaceStorage, namespaceStatusStorage, namespaceFinalizeStorage := namespaceetcd.NewStorage(c.DatabaseStorage)
m.namespaceRegistry = namespace.NewRegistry(namespaceStorage)
endpointsStorage := endpointsetcd.NewStorage(c.DatabaseStorage)
m.endpointRegistry = endpoint.NewRegistry(endpointsStorage)
nodeStorage, nodeStatusStorage := nodeetcd.NewStorage(c.DatabaseStorage, c.KubeletClient)
m.nodeRegistry = minion.NewRegistry(nodeStorage)
serviceStorage := serviceetcd.NewStorage(c.DatabaseStorage)
m.serviceRegistry = service.NewRegistry(serviceStorage)
var serviceClusterIPRegistry service.RangeRegistry
serviceClusterIPAllocator := ipallocator.NewAllocatorCIDRRange(m.serviceClusterIPRange, func(max int, rangeSpec string) allocator.Interface {
mem := allocator.NewAllocationMap(max, rangeSpec)
etcd := etcdallocator.NewEtcd(mem, "/ranges/serviceips", "serviceipallocation", c.DatabaseStorage)
serviceClusterIPRegistry = etcd
return etcd
})
m.serviceClusterIPAllocator = serviceClusterIPRegistry
var serviceNodePortRegistry service.RangeRegistry
serviceNodePortAllocator := portallocator.NewPortAllocatorCustom(m.serviceNodePortRange, func(max int, rangeSpec string) allocator.Interface {
mem := allocator.NewAllocationMap(max, rangeSpec)
etcd := etcdallocator.NewEtcd(mem, "/ranges/servicenodeports", "servicenodeportallocation", c.DatabaseStorage)
serviceNodePortRegistry = etcd
return etcd
})
m.serviceNodePortAllocator = serviceNodePortRegistry
controllerStorage := controlleretcd.NewREST(c.DatabaseStorage)
// TODO: Factor out the core API registration
m.storage = map[string]rest.Storage{
"pods": podStorage.Pod,
"pods/attach": podStorage.Attach,
"pods/status": podStorage.Status,
"pods/log": podStorage.Log,
"pods/exec": podStorage.Exec,
"pods/portforward": podStorage.PortForward,
"pods/proxy": podStorage.Proxy,
"pods/binding": podStorage.Binding,
"bindings": podStorage.Binding,
"podTemplates": podTemplateStorage,
"replicationControllers": controllerStorage,
"services": service.NewStorage(m.serviceRegistry, m.endpointRegistry, serviceClusterIPAllocator, serviceNodePortAllocator),
"endpoints": endpointsStorage,
"nodes": nodeStorage,
"nodes/status": nodeStatusStorage,
"events": event.NewStorage(eventRegistry),
"limitRanges": limitRangeStorage,
"resourceQuotas": resourceQuotaStorage,
"resourceQuotas/status": resourceQuotaStatusStorage,
"namespaces": namespaceStorage,
"namespaces/status": namespaceStatusStorage,
"namespaces/finalize": namespaceFinalizeStorage,
"secrets": secretStorage,
"serviceAccounts": serviceAccountStorage,
"persistentVolumes": persistentVolumeStorage,
"persistentVolumes/status": persistentVolumeStatusStorage,
"persistentVolumeClaims": persistentVolumeClaimStorage,
"persistentVolumeClaims/status": persistentVolumeClaimStatusStorage,
"componentStatuses": componentstatus.NewStorage(func() map[string]apiserver.Server { return m.getServersToValidate(c) }),
}
// establish the node proxy dialer
if len(c.SSHUser) > 0 {
// Usernames are capped @ 32
if len(c.SSHUser) > 32 {
glog.Warning("SSH User is too long, truncating to 32 chars")
c.SSHUser = c.SSHUser[0:32]
}
glog.Infof("Setting up proxy: %s %s", c.SSHUser, c.SSHKeyfile)
// public keyfile is written last, so check for that.
publicKeyFile := c.SSHKeyfile + ".pub"
exists, err := util.FileExists(publicKeyFile)
if err != nil {
glog.Errorf("Error detecting if key exists: %v", err)
} else if !exists {
glog.Infof("Key doesn't exist, attempting to create")
err := m.generateSSHKey(c.SSHUser, c.SSHKeyfile, publicKeyFile)
if err != nil {
glog.Errorf("Failed to create key pair: %v", err)
}
}
m.tunnels = &util.SSHTunnelList{}
m.dialer = m.Dial
m.setupSecureProxy(c.SSHUser, c.SSHKeyfile, publicKeyFile)
m.lastSync = m.clock.Now().Unix()
// This is pretty ugly. A better solution would be to pull this all the way up into the
// server.go file.
httpKubeletClient, ok := c.KubeletClient.(*client.HTTPKubeletClient)
if ok {
httpKubeletClient.Config.Dial = m.dialer
transport, err := client.MakeTransport(httpKubeletClient.Config)
if err != nil {
glog.Errorf("Error setting up transport over SSH: %v", err)
} else {
httpKubeletClient.Client.Transport = transport
}
} else {
glog.Errorf("Failed to cast %v to HTTPKubeletClient, skipping SSH tunnel.", c.KubeletClient)
}
healthzChecks = append(healthzChecks, healthz.NamedCheck("SSH Tunnel Check", m.IsTunnelSyncHealthy))
m.lastSyncMetric = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Name: "apiserver_proxy_tunnel_sync_latency_secs",
Help: "The time since the last successful synchronization of the SSH tunnels for proxy requests.",
}, func() float64 { return float64(m.secondsSinceSync()) })
}
apiVersions := []string{}
if m.v1 {
if err := m.api_v1().InstallREST(m.handlerContainer); err != nil {
glog.Fatalf("Unable to setup API v1: %v", err)
}
apiVersions = append(apiVersions, "v1")
}
apiserver.InstallSupport(m.muxHelper, m.rootWebService, c.EnableProfiling, healthzChecks...)
apiserver.AddApiWebService(m.handlerContainer, c.APIPrefix, apiVersions)
defaultVersion := m.defaultAPIGroupVersion()
requestInfoResolver := &apiserver.APIRequestInfoResolver{APIPrefixes: util.NewStringSet(strings.TrimPrefix(defaultVersion.Root, "/")), RestMapper: defaultVersion.Mapper}
apiserver.InstallServiceErrorHandler(m.handlerContainer, requestInfoResolver, apiVersions)
if m.exp {
expVersion := m.expapi(c)
if err := expVersion.InstallREST(m.handlerContainer); err != nil {
glog.Fatalf("Unable to setup experimental api: %v", err)
}
apiserver.AddApiWebService(m.handlerContainer, c.ExpAPIPrefix, []string{expVersion.Version})
expRequestInfoResolver := &apiserver.APIRequestInfoResolver{APIPrefixes: util.NewStringSet(strings.TrimPrefix(expVersion.Root, "/")), RestMapper: expVersion.Mapper}
apiserver.InstallServiceErrorHandler(m.handlerContainer, expRequestInfoResolver, []string{expVersion.Version})
}
// Register root handler.
// We do not register this using restful Webservice since we do not want to surface this in api docs.
// Allow master to be embedded in contexts which already have something registered at the root
if c.EnableIndex {
m.mux.HandleFunc("/", apiserver.IndexHandler(m.handlerContainer, m.muxHelper))
}
if c.EnableLogsSupport {
apiserver.InstallLogsSupport(m.muxHelper)
}
if c.EnableUISupport {
ui.InstallSupport(m.muxHelper, m.enableSwaggerSupport)
}
if c.EnableProfiling {
m.mux.HandleFunc("/debug/pprof/", pprof.Index)
m.mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
m.mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
}
handler := http.Handler(m.mux.(*http.ServeMux))
// TODO: handle CORS and auth using go-restful
// See github.com/emicklei/go-restful/blob/master/examples/restful-CORS-filter.go, and
// github.com/emicklei/go-restful/blob/master/examples/restful-basic-authentication.go
if len(c.CorsAllowedOriginList) > 0 {
allowedOriginRegexps, err := util.CompileRegexps(c.CorsAllowedOriginList)
if err != nil {
glog.Fatalf("Invalid CORS allowed origin, --cors-allowed-origins flag was set to %v - %v", strings.Join(c.CorsAllowedOriginList, ","), err)
}
handler = apiserver.CORS(handler, allowedOriginRegexps, nil, nil, "true")
}
m.InsecureHandler = handler
attributeGetter := apiserver.NewRequestAttributeGetter(m.requestContextMapper, latest.RESTMapper, "api")
handler = apiserver.WithAuthorizationCheck(handler, attributeGetter, m.authorizer)
// Install Authenticator
if c.Authenticator != nil {
authenticatedHandler, err := handlers.NewRequestAuthenticator(m.requestContextMapper, c.Authenticator, handlers.Unauthorized(c.SupportsBasicAuth), handler)
if err != nil {
glog.Fatalf("Could not initialize authenticator: %v", err)
}
handler = authenticatedHandler
}
// Install root web services
m.handlerContainer.Add(m.rootWebService)
// TODO: Make this optional? Consumers of master depend on this currently.
m.Handler = handler
if m.enableSwaggerSupport {
m.InstallSwaggerAPI()
}
// After all wrapping is done, put a context filter around both handlers
if handler, err := api.NewRequestContextFilter(m.requestContextMapper, m.Handler); err != nil {
glog.Fatalf("Could not initialize request context filter: %v", err)
} else {
m.Handler = handler
}
if handler, err := api.NewRequestContextFilter(m.requestContextMapper, m.InsecureHandler); err != nil {
glog.Fatalf("Could not initialize request context filter: %v", err)
} else {
m.InsecureHandler = handler
}
// TODO: Attempt clean shutdown?
if m.enableCoreControllers {
m.NewBootstrapController().Start()
}
}
// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (c *cinderVolumeUnmounter) TearDownAt(dir string) error {
glog.V(5).Infof("Cinder TearDown of %s", dir)
if _, err := os.Stat(dir); os.IsNotExist(err) {
// non-exist dir for TearDown is meaningless and it is possible that this dir has been cleaned up, just omit the error for now
glog.Warningf("Volume directory: %v does not exists, it may have been cleaned up by previous tear down task", dir)
return nil
}
notmnt, err := c.mounter.IsLikelyNotMountPoint(dir)
if err != nil {
glog.V(4).Infof("IsLikelyNotMountPoint check failed: %v", err)
return err
}
exist, _ := util.FileExists(path.Join(dir, OpenStackCloudProviderTagFile))
if exist {
c.withOpenStackCP = false
} else {
c.withOpenStackCP = true
}
if notmnt {
// Find Cinder volumeID to lock the right volume
// TODO: refactor VolumePlugin.NewCleaner to get full volume.Spec just like
// NewBuilder. We could then find volumeID there without probing MountRefs.
if !c.withOpenStackCP && c.isNoMountSupported {
volumeID, err := ioutil.ReadFile(path.Join(dir, OpenStackCloudProviderTagFile))
if err != nil {
return err
}
c.pdName = string(volumeID)
if err := c.manager.DetachDisk(c); err != nil {
return err
}
}
return os.RemoveAll(dir)
}
// Find Cinder volumeID to lock the right volume
// TODO: refactor VolumePlugin.NewUnmounter to get full volume.Spec just like
// NewMounter. We could then find volumeID there without probing MountRefs.
refs, err := mount.GetMountRefs(c.mounter, dir)
if err != nil {
glog.V(4).Infof("GetMountRefs failed: %v", err)
return err
}
if len(refs) == 0 {
glog.V(4).Infof("Directory %s is not mounted", dir)
return fmt.Errorf("directory %s is not mounted", dir)
}
c.pdName = path.Base(refs[0])
glog.V(4).Infof("Found volume %s mounted to %s", c.pdName, dir)
// lock the volume (and thus wait for any concurrrent SetUpAt to finish)
c.plugin.volumeLocks.LockKey(c.pdName)
defer c.plugin.volumeLocks.UnlockKey(c.pdName)
// Reload list of references, there might be SetUpAt finished in the meantime
refs, err = mount.GetMountRefs(c.mounter, dir)
if err != nil {
glog.V(4).Infof("GetMountRefs failed: %v", err)
return err
}
if err := c.mounter.Unmount(dir); err != nil {
glog.V(4).Infof("Unmount failed: %v", err)
return err
}
glog.V(3).Infof("Successfully unmounted: %s\n", dir)
notmnt, mntErr := c.mounter.IsLikelyNotMountPoint(dir)
if mntErr != nil {
glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
return err
}
if notmnt {
if err := os.Remove(dir); err != nil {
glog.V(4).Infof("Failed to remove directory after unmount: %v", err)
return err
}
}
return nil
}
// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (c *cinderVolumeCleaner) TearDownAt(dir string) error {
glog.V(5).Infof("Cinder TearDown of %s", dir)
notmnt, err := c.mounter.IsLikelyNotMountPoint(dir)
if err != nil {
glog.V(4).Infof("IsLikelyNotMountPoint check failed: %v", err)
return err
}
exist, _ := util.FileExists(path.Join(dir, OpenStackCloudProviderTagFile))
if exist {
c.withOpenStackCP = false
} else {
c.withOpenStackCP = true
}
if notmnt {
// Find Cinder volumeID to lock the right volume
// TODO: refactor VolumePlugin.NewCleaner to get full volume.Spec just like
// NewBuilder. We could then find volumeID there without probing MountRefs.
if !c.withOpenStackCP && c.isNoMountSupported {
volumeID, err := ioutil.ReadFile(path.Join(dir, OpenStackCloudProviderTagFile))
if err != nil {
return err
}
c.pdName = string(volumeID)
if err := c.manager.DetachDisk(c); err != nil {
return err
}
}
return os.RemoveAll(dir)
}
refs, err := mount.GetMountRefs(c.mounter, dir)
if err != nil {
glog.V(4).Infof("GetMountRefs failed: %v", err)
return err
}
if len(refs) == 0 {
glog.V(4).Infof("Directory %s is not mounted", dir)
return fmt.Errorf("directory %s is not mounted", dir)
}
c.pdName = path.Base(refs[0])
glog.V(4).Infof("Found volume %s mounted to %s", c.pdName, dir)
// lock the volume (and thus wait for any concurrrent SetUpAt to finish)
c.plugin.volumeLocks.LockKey(c.pdName)
defer c.plugin.volumeLocks.UnlockKey(c.pdName)
// Reload list of references, there might be SetUpAt finished in the meantime
refs, err = mount.GetMountRefs(c.mounter, dir)
if err != nil {
glog.V(4).Infof("GetMountRefs failed: %v", err)
return err
}
if err := c.mounter.Unmount(dir); err != nil {
glog.V(4).Infof("Unmount failed: %v", err)
return err
}
glog.V(3).Infof("Successfully unmounted: %s\n", dir)
// If refCount is 1, then all bind mounts have been removed, and the
// remaining reference is the global mount. It is safe to detach.
if len(refs) == 1 {
if err := c.manager.DetachDisk(c); err != nil {
glog.V(4).Infof("DetachDisk failed: %v", err)
return err
}
glog.V(3).Infof("Volume %s detached", c.pdName)
}
notmnt, mntErr := c.mounter.IsLikelyNotMountPoint(dir)
if mntErr != nil {
glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
return err
}
if notmnt {
if err := os.Remove(dir); err != nil {
glog.V(4).Infof("Failed to remove directory after unmount: %v", err)
return err
}
}
return nil
}