func (c *client) getExecutorChecksum(ctx types.Context) (string, error) {
if c.isController() {
return "", utils.NewUnsupportedForClientTypeError(
c.clientType, "getExecutorChecksum")
}
ctx.Debug("getting executor checksum")
f, err := os.Open(types.LSX.String())
if err != nil {
return "", err
}
defer f.Close()
h := md5.New()
buf := make([]byte, 1024)
for {
n, err := f.Read(buf)
if err == io.EOF {
break
}
if err != nil {
return "", err
}
if _, err := h.Write(buf[:n]); err != nil {
return "", err
}
}
sum := fmt.Sprintf("%x", h.Sum(nil))
ctx.WithField("localChecksum", sum).Debug("got local executor checksum")
return sum, nil
}
// initDefaultLibStorageServices initializes the config object with a default
// libStorage service if one is not present.
//
// TODO Move this into libStorage in libStorage 0.1.2
func initDefaultLibStorageServices(
ctx apitypes.Context, config gofig.Config) error {
if config.IsSet(apitypes.ConfigServices) {
ctx.Debug(
"libStorage auto service mode disabled; services defined")
return nil
}
serviceName := config.GetString(apitypes.ConfigService)
if serviceName == "" {
ctx.Debug(
"libStorage auto service mode disabled; service name empty")
return nil
}
ctx.WithField("driver", serviceName).Info(
"libStorage auto service mode enabled")
buf := &bytes.Buffer{}
fmt.Fprintf(buf, defaultServiceConfigFormat, serviceName)
if err := config.ReadConfig(buf); err != nil {
return err
}
return nil
}
func (c *client) downloadExecutor(ctx types.Context) error {
if c.isController() {
return utils.NewUnsupportedForClientTypeError(
c.clientType, "downloadExecutor")
}
ctx.Debug("downloading executor")
f, err := os.OpenFile(
types.LSX.String(),
os.O_CREATE|os.O_RDWR|os.O_TRUNC,
0755)
if err != nil {
return err
}
defer f.Close()
rdr, err := c.APIClient.ExecutorGet(ctx, types.LSX.Name())
n, err := io.Copy(f, rdr)
if err != nil {
return err
}
if err := f.Sync(); err != nil {
return err
}
ctx.WithField("bytes", n).Debug("downloaded executor")
return nil
}
func (c *client) NextDevice(
ctx types.Context,
opts types.Store) (string, error) {
if c.isController() {
return "", utils.NewUnsupportedForClientTypeError(
c.clientType, "NextDevice")
}
if supported, _ := c.Supported(ctx, opts); !supported {
return "", errExecutorNotSupported
}
ctx = context.RequireTX(ctx.Join(c.ctx))
serviceName, ok := context.ServiceName(ctx)
if !ok {
return "", goof.New("missing service name")
}
si, err := c.getServiceInfo(serviceName)
if err != nil {
return "", err
}
driverName := si.Driver.Name
out, err := c.runExecutor(ctx, driverName, types.LSXCmdNextDevice)
if err != nil {
return "", err
}
ctx.Debug("xli nextdevice success")
return gotil.Trim(string(out)), nil
}
func (c *client) runExecutor(
ctx types.Context, args ...string) ([]byte, error) {
if c.isController() {
return nil, utils.NewUnsupportedForClientTypeError(
c.clientType, "runExecutor")
}
ctx.Debug("waiting on executor lock")
if err := c.lsxMutexWait(); err != nil {
return nil, err
}
defer func() {
ctx.Debug("signalling executor lock")
if err := c.lsxMutexSignal(); err != nil {
panic(err)
}
}()
lsxBin := types.LSX.String()
cmd := exec.Command(lsxBin, args...)
cmd.Env = os.Environ()
configEnvVars := c.config.EnvVars()
for _, cev := range configEnvVars {
// ctx.WithField("value", cev).Debug("set executor env var")
cmd.Env = append(cmd.Env, cev)
}
out, err := cmd.Output()
if exitError, ok := err.(*exec.ExitError); ok {
exitCode := exitError.Sys().(syscall.WaitStatus).ExitStatus()
switch exitCode {
case types.LSXExitCodeNotImplemented:
return nil, types.ErrNotImplemented
case types.LSXExitCodeTimedOut:
return nil, types.ErrTimedOut
}
return nil, goof.WithFieldsE(
map[string]interface{}{
"lsx": lsxBin,
"args": args,
},
"error executing xcli",
err)
}
return out, err
}
func waitUntilLibStorageStopped(ctx apitypes.Context, errs <-chan error) {
ctx.Debug("waiting until libStorage is stopped")
// if there is no err channel then do not wait until libStorage is stopped
// as the absence of the err channel means libStorage was not started in
// embedded mode
if errs == nil {
ctx.Debug("done waiting on err chan; err chan is nil")
return
}
// in a goroutine, range over the apiserver.Close channel until it's closed
for range apiserver.Close() {
}
ctx.Debug("done sending close signals to libStorage")
// block until the err channel is closed
for err := range errs {
if err == nil {
continue
}
ctx.WithError(err).Error("error on closing libStorage server")
}
ctx.Debug("done waiting on err chan")
}
func (d *idm) Remove(
ctx types.Context,
volumeName string,
opts types.Store) error {
fields := log.Fields{
"volumeName": volumeName,
"opts": opts}
ctx.WithFields(fields).Debug("removing volume")
if d.disableRemove() {
ctx.Debug("disableRemove skipped deletion")
return nil
}
return d.IntegrationDriver.Remove(ctx.Join(d.ctx), volumeName, opts)
}
func (d *idm) Create(
ctx types.Context,
volumeName string,
opts *types.VolumeCreateOpts) (*types.Volume, error) {
fields := log.Fields{
"volumeName": volumeName,
"opts": opts}
ctx.WithFields(fields).Debug("creating volume")
if d.disableCreate() {
ctx.Debug("disableRemove skipped creation")
return nil, nil
}
return d.IntegrationDriver.Create(ctx.Join(d.ctx), volumeName, opts)
}
func (c *client) updateExecutor(ctx types.Context) error {
if c.isController() {
return utils.NewUnsupportedForClientTypeError(
c.clientType, "updateExecutor")
}
ctx.Debug("updating executor")
lsxi := c.lsxCache.GetExecutorInfo(types.LSX.Name())
if lsxi == nil {
return goof.WithField("lsx", types.LSX, "unknown executor")
}
ctx.Debug("waiting on executor lock")
if err := c.lsxMutexWait(); err != nil {
return err
}
defer func() {
ctx.Debug("signalling executor lock")
if err := c.lsxMutexSignal(); err != nil {
panic(err)
}
}()
if !types.LSX.Exists() {
ctx.Debug("executor does not exist, download executor")
return c.downloadExecutor(ctx)
}
ctx.Debug("executor exists, getting local checksum")
checksum, err := c.getExecutorChecksum(ctx)
if err != nil {
return err
}
if lsxi.MD5Checksum != checksum {
ctx.WithFields(log.Fields{
"remoteChecksum": lsxi.MD5Checksum,
"localChecksum": checksum,
}).Debug("executor checksums do not match, download executor")
return c.downloadExecutor(ctx)
}
return nil
}
func (c *client) WaitForDevice(
ctx types.Context,
opts *types.WaitForDeviceOpts) (bool, *types.LocalDevices, error) {
if c.isController() {
return false, nil, utils.NewUnsupportedForClientTypeError(
c.clientType, "WaitForDevice")
}
if supported, _ := c.Supported(ctx, opts.Opts); !supported {
return false, nil, errExecutorNotSupported
}
ctx = context.RequireTX(ctx.Join(c.ctx))
serviceName, ok := context.ServiceName(ctx)
if !ok {
return false, nil, goof.New("missing service name")
}
si, err := c.getServiceInfo(serviceName)
if err != nil {
return false, nil, err
}
driverName := si.Driver.Name
out, err := c.runExecutor(
ctx, driverName, types.LSXCmdWaitForDevice,
opts.ScanType.String(), opts.Token, opts.Timeout.String())
if err != types.ErrTimedOut {
return false, nil, err
}
matched := err == nil
ld, err := unmarshalLocalDevices(ctx, out)
if err != nil {
return false, nil, err
}
ctx.Debug("xli waitfordevice success")
return matched, ld, nil
}
func (c *client) LocalDevices(
ctx types.Context,
opts *types.LocalDevicesOpts) (*types.LocalDevices, error) {
if c.isController() {
return nil, utils.NewUnsupportedForClientTypeError(
c.clientType, "LocalDevices")
}
if supported, _ := c.Supported(ctx, opts.Opts); !supported {
return nil, errExecutorNotSupported
}
ctx = context.RequireTX(ctx.Join(c.ctx))
serviceName, ok := context.ServiceName(ctx)
if !ok {
return nil, goof.New("missing service name")
}
si, err := c.getServiceInfo(serviceName)
if err != nil {
return nil, err
}
driverName := si.Driver.Name
out, err := c.runExecutor(
ctx, driverName, types.LSXCmdLocalDevices, opts.ScanType.String())
if err != nil {
return nil, err
}
ld, err := unmarshalLocalDevices(ctx, out)
if err != nil {
return nil, err
}
ctx.Debug("xli localdevices success")
return ld, nil
}
// Mount will return a mount point path when specifying either a volumeName
// or volumeID. If a overwriteFs boolean is specified it will overwrite
// the FS based on newFsType if it is detected that there is no FS present.
func (d *driver) Mount(
ctx types.Context,
volumeID, volumeName string,
opts *types.VolumeMountOpts) (string, *types.Volume, error) {
ctx.WithFields(log.Fields{
"volumeName": volumeName,
"volumeID": volumeID,
"opts": opts}).Info("mounting volume")
vol, err := d.volumeInspectByIDOrName(
ctx, volumeID, volumeName,
types.VolAttReqWithDevMapOnlyVolsAttachedToInstanceOrUnattachedVols,
opts.Opts)
if isErrNotFound(err) && d.volumeCreateImplicit() {
var err error
if vol, err = d.Create(ctx, volumeName, &types.VolumeCreateOpts{
Opts: utils.NewStore(),
}); err != nil {
return "", nil, goof.WithError(
"problem creating volume implicitly", err)
}
} else if err != nil {
return "", nil, err
}
if vol == nil {
return "", nil, goof.New("no volume returned or created")
}
client := context.MustClient(ctx)
if len(vol.Attachments) == 0 || opts.Preempt {
mp, err := d.getVolumeMountPath(vol.Name)
if err != nil {
return "", nil, err
}
ctx.Debug("performing precautionary unmount")
_ = client.OS().Unmount(ctx, mp, opts.Opts)
var token string
vol, token, err = client.Storage().VolumeAttach(
ctx, vol.ID, &types.VolumeAttachOpts{
Force: opts.Preempt,
Opts: utils.NewStore(),
})
if err != nil {
return "", nil, err
}
if token != "" {
opts := &types.WaitForDeviceOpts{
LocalDevicesOpts: types.LocalDevicesOpts{
ScanType: apiconfig.DeviceScanType(d.config),
Opts: opts.Opts,
},
Token: token,
Timeout: apiconfig.DeviceAttachTimeout(d.config),
}
_, _, err = client.Executor().WaitForDevice(ctx, opts)
if err != nil {
return "", nil, goof.WithError(
"problem with device discovery", err)
}
}
vol, err = d.volumeInspectByIDOrName(
ctx, vol.ID, "", types.VolAttReqTrue, opts.Opts)
if err != nil {
return "", nil, err
}
}
if len(vol.Attachments) == 0 {
return "", nil, goof.New("volume did not attach")
}
inst, err := client.Storage().InstanceInspect(ctx, utils.NewStore())
if err != nil {
return "", nil, goof.New("problem getting instance ID")
}
var ma *types.VolumeAttachment
for _, att := range vol.Attachments {
if att.InstanceID.ID == inst.InstanceID.ID {
ma = att
break
}
}
if ma == nil {
return "", nil, goof.New("no local attachment found")
}
if ma.DeviceName == "" {
return "", nil, goof.New("no device name returned")
}
mounts, err := client.OS().Mounts(
ctx, ma.DeviceName, "", opts.Opts)
if err != nil {
return "", nil, err
}
if len(mounts) > 0 {
return d.volumeMountPath(mounts[0].MountPoint), vol, nil
}
if opts.NewFSType == "" {
opts.NewFSType = d.fsType()
}
if err := client.OS().Format(
ctx,
ma.DeviceName,
&types.DeviceFormatOpts{
NewFSType: opts.NewFSType,
OverwriteFS: opts.OverwriteFS,
}); err != nil {
return "", nil, err
}
mountPath, err := d.getVolumeMountPath(vol.Name)
if err != nil {
return "", nil, err
}
if err := os.MkdirAll(mountPath, 0755); err != nil {
return "", nil, err
}
if err := client.OS().Mount(
ctx,
ma.DeviceName,
mountPath,
&types.DeviceMountOpts{}); err != nil {
return "", nil, err
}
mntPath := d.volumeMountPath(mountPath)
fields := log.Fields{
"vol": vol,
"mntPath": mntPath,
}
ctx.WithFields(fields).Info("volume mounted")
return mntPath, vol, nil
}
// Used in VolumeCreate
func (d *driver) createVolume(
ctx types.Context,
volumeName, snapshotID string,
opts *types.VolumeCreateOpts) (*awsec2.Volume, error) {
var (
err error
server awsec2.Instance
)
// Create volume using EC2 API call
if server, err = d.getInstance(ctx); err != nil {
return &awsec2.Volume{}, goof.WithError(
"error creating volume with EC2 API call", err)
}
// Fill in Availability Zone if needed
d.createVolumeEnsureAvailabilityZone(opts.AvailabilityZone, &server)
options := &awsec2.CreateVolumeInput{
Size: opts.Size,
AvailabilityZone: opts.AvailabilityZone,
Encrypted: opts.Encrypted,
VolumeType: opts.Type,
}
if snapshotID != "" {
options.SnapshotId = &snapshotID
}
if opts.IOPS != nil && *opts.IOPS > 0 {
options.Iops = opts.IOPS
}
if opts.Encrypted != nil && *opts.Encrypted {
if opts.EncryptionKey != nil && len(*opts.EncryptionKey) > 0 {
ctx.Debug("creating encrypted volume w client enc key")
options.KmsKeyId = opts.EncryptionKey
} else if len(d.kmsKeyID) > 0 {
ctx.Debug("creating encrypted volume w server enc key")
options.KmsKeyId = aws.String(d.kmsKeyID)
} else {
ctx.Debug("creating encrypted volume w default enc key")
}
}
var resp *awsec2.Volume
if resp, err = mustSession(ctx).CreateVolume(options); err != nil {
return &awsec2.Volume{}, goof.WithError(
"error creating volume", err)
}
// Add tags to created volume
if err = d.createTags(ctx, *resp.VolumeId, volumeName); err != nil {
return &awsec2.Volume{}, goof.WithError(
"error creating tags", err)
}
// Wait for volume status to change
if err = d.waitVolumeComplete(
ctx, *resp.VolumeId, waitVolumeCreate); err != nil {
return &awsec2.Volume{}, goof.WithError(
"error waiting for volume creation", err)
}
return resp, nil
}
func (c *client) InstanceID(
ctx types.Context,
opts types.Store) (*types.InstanceID, error) {
if c.isController() {
return nil, utils.NewUnsupportedForClientTypeError(
c.clientType, "InstanceID")
}
if supported, _ := c.Supported(ctx, opts); !supported {
return nil, errExecutorNotSupported
}
ctx = context.RequireTX(ctx.Join(c.ctx))
serviceName, ok := context.ServiceName(ctx)
if !ok {
return nil, goof.New("missing service name")
}
si, err := c.getServiceInfo(serviceName)
if err != nil {
return nil, err
}
driverName := strings.ToLower(si.Driver.Name)
// check to see if the driver's instance ID is cached
if iid := c.instanceIDCache.GetInstanceID(driverName); iid != nil {
return iid, nil
}
out, err := c.runExecutor(ctx, driverName, types.LSXCmdInstanceID)
if err != nil {
return nil, err
}
iid := &types.InstanceID{}
if err := iid.UnmarshalText(out); err != nil {
return nil, err
}
ctx = ctx.WithValue(context.InstanceIDKey, iid)
if iid.HasMetadata() {
ctx.Debug("sending instanceID in API.InstanceInspect call")
instance, err := c.InstanceInspect(ctx, serviceName)
if err != nil {
return nil, err
}
ctx.Debug("received instanceID from API.InstanceInspect call")
iid.ID = instance.InstanceID.ID
iid.Fields = instance.InstanceID.Fields
iid.DeleteMetadata()
}
c.instanceIDCache.Set(driverName, iid)
ctx.Debug("cached instanceID")
ctx.Debug("xli instanceID success")
return iid, nil
}
func activateLibStorage(
ctx apitypes.Context,
config gofig.Config) (apitypes.Context, gofig.Config, <-chan error, error) {
apiserver.DisableStartupInfo = true
var (
host string
err error
isRunning bool
errs chan error
serverErrChan <-chan error
server apitypes.Server
)
if host = config.GetString(apitypes.ConfigHost); host != "" {
if !config.GetBool(apitypes.ConfigEmbedded) {
ctx.WithField(
"host", host,
).Debug("not starting embedded server; embedded mode disabled")
return ctx, config, nil, nil
}
}
if host, isRunning = IsLocalServerActive(ctx, config); isRunning {
ctx = setHost(ctx, config, host)
ctx.WithField("host", host).Debug(
"not starting embedded server; already running")
return ctx, config, nil, nil
}
// if no host was specified then see if a set of default services need to
// be initialized
if host == "" {
ctx.Debug("host is empty; initiliazing default services")
if err = initDefaultLibStorageServices(ctx, config); err != nil {
ctx.WithError(err).Error("error initializing default services")
return ctx, config, nil, err
}
}
ctx.Debug("starting embedded libStorage server")
if server, serverErrChan, err = apiserver.Serve(ctx, config); err != nil {
ctx.WithError(err).Error("error starting libStorage server")
return ctx, config, nil, err
}
if host == "" {
host = server.Addrs()[0]
ctx.WithField("host", host).Debug("got host from new server address")
}
ctx = setHost(ctx, config, host)
errs = make(chan error)
go func() {
for err := range serverErrChan {
if err != nil {
errs <- err
}
}
if err := os.RemoveAll(SpecFilePath()); err == nil {
logHostSpec(ctx, host, "removed spec file")
}
close(errs)
}()
// write the host to the spec file so that other rex-ray invocations can
// find it, even if running as an embedded libStorage server
if err := WriteSpecFile(host); err != nil {
specFile := SpecFilePath()
if os.IsPermission(err) {
ctx.WithError(err).Errorf(
"user does not have write permissions for %s", specFile)
} else {
ctx.WithError(err).Errorf(
"error writing spec file at %s", specFile)
}
//WaitUntilLibStorageStopped(ctx, serverErrChan)
return ctx, config, errs, err
}
logHostSpec(ctx, host, "created spec file")
return ctx, config, errs, nil
}