func (d *Dispatcher) reconfigureApplianceSpec(vm *vm.VirtualMachine, conf *metadata.VirtualContainerHostConfigSpec) (*types.VirtualMachineConfigSpec, error) {
defer trace.End(trace.Begin(""))
var devices object.VirtualDeviceList
var err error
spec := &types.VirtualMachineConfigSpec{
Name: conf.Name,
GuestId: "other3xLinux64Guest",
Files: &types.VirtualMachineFileInfo{VmPathName: fmt.Sprintf("[%s]", conf.ImageStores[0].Host)},
}
if devices, err = d.configIso(conf, vm); err != nil {
return nil, err
}
deviceChange, err := devices.ConfigSpec(types.VirtualDeviceConfigSpecOperationAdd)
if err != nil {
log.Errorf("Failed to create config spec for appliance: %s", err)
return nil, err
}
spec.DeviceChange = deviceChange
cfg := make(map[string]string)
extraconfig.Encode(extraconfig.MapSink(cfg), conf)
spec.ExtraConfig = append(spec.ExtraConfig, extraconfig.OptionValueFromMap(cfg)...)
return spec, nil
}
func (h *Handle) Commit(ctx context.Context, sess *session.Session, waitTime *int32) error {
cfg := make(map[string]string)
// Set timestamps based on target state
switch h.TargetState() {
case StateRunning:
for _, sc := range h.ExecConfig.Sessions {
sc.StartTime = time.Now().UTC().Unix()
sc.Started = ""
sc.ExitStatus = 0
}
case StateStopped:
for _, sc := range h.ExecConfig.Sessions {
sc.StopTime = time.Now().UTC().Unix()
}
}
extraconfig.Encode(extraconfig.MapSink(cfg), h.ExecConfig)
s := h.Spec.Spec()
s.ExtraConfig = append(s.ExtraConfig, vmomi.OptionValueFromMap(cfg)...)
if err := Commit(ctx, sess, h, waitTime); err != nil {
return err
}
removeHandle(h.key)
return nil
}
func StartAttachTether(t *testing.T, cfg *metadata.ExecutorConfig) (tether.Tether, extraconfig.DataSource, net.Conn) {
store := map[string]string{}
sink := extraconfig.MapSink(store)
src := extraconfig.MapSource(store)
extraconfig.Encode(sink, cfg)
log.Debugf("Test configuration: %#v", sink)
tthr := tether.New(src, sink, &Mocked)
tthr.Register("mocker", &Mocked)
tthr.Register("Attach", server)
// run the tether to service the attach
go func() {
erR := tthr.Start()
if erR != nil {
t.Error(erR)
}
}()
// create client on the mock pipe
conn, err := mockBackChannel(context.Background())
if err != nil && (err != io.EOF || server.(*testAttachServer).enabled) {
// we accept the case where the error is end-of-file and the attach server is disabled because that's
// expected when the tether is shut down.
t.Error(err)
}
return tthr, src, conn
}
func TestToExtraConfig(t *testing.T) {
exec := metadata.ExecutorConfig{
Common: metadata.Common{
ID: "deadbeef",
Name: "configtest",
},
Sessions: map[string]metadata.SessionConfig{
"deadbeef": metadata.SessionConfig{
Cmd: metadata.Cmd{
Path: "/bin/bash",
Args: []string{"/bin/bash", "-c", "echo hello"},
Dir: "/",
Env: []string{"HOME=/", "PATH=/bin"},
},
},
"beefed": metadata.SessionConfig{
Cmd: metadata.Cmd{
Path: "/bin/bash",
Args: []string{"/bin/bash", "-c", "echo goodbye"},
Dir: "/",
Env: []string{"HOME=/", "PATH=/bin"},
},
},
},
Networks: map[string]*metadata.NetworkEndpoint{
"eth0": &metadata.NetworkEndpoint{
Static: &net.IPNet{IP: localhost, Mask: lmask.Mask},
Network: metadata.ContainerNetwork{
Common: metadata.Common{
Name: "notsure",
},
Gateway: net.IPNet{IP: gateway, Mask: gmask.Mask},
Nameservers: []net.IP{},
},
},
},
}
// encode metadata package's ExecutorConfig
encoded := map[string]string{}
extraconfig.Encode(extraconfig.MapSink(encoded), exec)
// decode into this package's ExecutorConfig
var decoded ExecutorConfig
extraconfig.Decode(extraconfig.MapSource(encoded), &decoded)
// the networks should be identical
assert.Equal(t, exec.Networks["eth0"], decoded.Networks["eth0"])
// the source and destination structs are different - we're doing a sparse comparison
expected := exec.Sessions["deadbeef"]
actual := *decoded.Sessions["deadbeef"]
assert.Equal(t, expected.Cmd.Path, actual.Cmd.Path)
assert.Equal(t, expected.Cmd.Args, actual.Cmd.Args)
assert.Equal(t, expected.Cmd.Dir, actual.Cmd.Dir)
assert.Equal(t, expected.Cmd.Env, actual.Cmd.Env)
}
func testConfig() *Configuration {
return &Configuration{
source: extraconfig.MapSource(map[string]string{}),
sink: extraconfig.MapSink(map[string]string{}),
BridgeLink: &mockLink{},
Network: config.Network{
BridgeNetwork: "bridge",
ContainerNetworks: map[string]*executor.ContainerNetwork{
"bridge": {
Common: executor.Common{
Name: "bridge",
},
Type: constants.BridgeScopeType,
},
"bar7": {
Common: executor.Common{
Name: "external",
},
Gateway: net.IPNet{IP: net.ParseIP("10.13.0.1"), Mask: net.CIDRMask(16, 32)},
Nameservers: []net.IP{net.ParseIP("10.10.1.1")},
Pools: []ip.Range{*ip.ParseRange("10.13.1.0-255"), *ip.ParseRange("10.13.2.0-10.13.2.15")},
Type: constants.ExternalScopeType,
},
"bar71": {
Common: executor.Common{
Name: "external",
},
Gateway: net.IPNet{IP: net.ParseIP("10.131.0.1"), Mask: net.CIDRMask(16, 32)},
Nameservers: []net.IP{net.ParseIP("10.131.0.1"), net.ParseIP("10.131.0.2")},
Pools: []ip.Range{*ip.ParseRange("10.131.1.0/16")},
Type: constants.ExternalScopeType,
},
"bar72": {
Common: executor.Common{
Name: "external",
},
Type: constants.ExternalScopeType,
},
"bar73": {
Common: executor.Common{
Name: "external",
},
Gateway: net.IPNet{IP: net.ParseIP("10.133.0.1"), Mask: net.CIDRMask(16, 32)},
Type: constants.ExternalScopeType,
},
},
},
PortGroups: map[string]object.NetworkReference{
"bridge": testBridgeNetwork,
"bar7": testExternalNetwork,
"bar71": testExternalNetwork,
"bar72": testExternalNetwork,
"bar73": testExternalNetwork,
},
}
}
func logConfig(config *ExecutorConfig) {
// just pretty print the json for now
log.Info("Loaded executor config")
if log.GetLevel() == log.DebugLevel && config.DebugLevel > 1 {
sink := map[string]string{}
extraconfig.Encode(extraconfig.MapSink(sink), config)
for k, v := range sink {
log.Debugf("%s: %s", k, v)
}
}
}
func RunTether(t *testing.T, cfg *metadata.ExecutorConfig) (Tether, extraconfig.DataSource, error) {
store := map[string]string{}
sink := extraconfig.MapSink(store)
src := extraconfig.MapSource(store)
extraconfig.Encode(sink, cfg)
log.Debugf("Test configuration: %#v", sink)
tthr := New(src, sink, &Mocked)
tthr.Register("Mocker", &Mocked)
// run the tether to service the attach
erR := tthr.Start()
return tthr, src, erR
}
func logConfig(config *ExecutorConfig) {
// just pretty print the json for now
log.Info("Loaded executor config")
// TODO: investigate whether it's the govmomi types package cause the binary size
// inflation - if so we need an alternative approach here or in extraconfig
if log.GetLevel() == log.DebugLevel {
sink := map[string]string{}
extraconfig.Encode(extraconfig.MapSink(sink), config)
for k, v := range sink {
log.Debugf("%s: %s", k, v)
}
}
}
func (d *Dispatcher) encodeConfig(conf *config.VirtualContainerHostConfigSpec) (map[string]string, error) {
if d.secret == nil {
log.Debug("generating new config secret key")
s, err := extraconfig.NewSecretKey()
if err != nil {
return nil, err
}
d.secret = s
}
cfg := make(map[string]string)
extraconfig.Encode(d.secret.Sink(extraconfig.MapSink(cfg)), conf)
return cfg, nil
}
func (d *Dispatcher) createApplianceSpec(conf *metadata.VirtualContainerHostConfigSpec, vConf *data.InstallerData) (*types.VirtualMachineConfigSpec, error) {
defer trace.End(trace.Begin(""))
var devices object.VirtualDeviceList
var err error
cfg := make(map[string]string)
extraconfig.Encode(extraconfig.MapSink(cfg), conf)
spec := &spec.VirtualMachineConfigSpec{
VirtualMachineConfigSpec: &types.VirtualMachineConfigSpec{
Name: conf.Name,
GuestId: "other3xLinux64Guest",
Files: &types.VirtualMachineFileInfo{VmPathName: fmt.Sprintf("[%s]", conf.ImageStores[0].Host)},
NumCPUs: int32(vConf.ApplianceSize.CPU.Limit),
MemoryMB: vConf.ApplianceSize.Memory.Limit,
// Encode the config both here and after the VMs created so that it can be identified as a VCH appliance as soon as
// creation is complete.
ExtraConfig: extraconfig.OptionValueFromMap(cfg),
},
}
if devices, err = d.addIDEController(devices); err != nil {
return nil, err
}
if devices, err = d.addParaVirtualSCSIController(devices); err != nil {
return nil, err
}
if devices, err = d.addNetworkDevices(conf, spec, devices); err != nil {
return nil, err
}
deviceChange, err := devices.ConfigSpec(types.VirtualDeviceConfigSpecOperationAdd)
if err != nil {
return nil, err
}
spec.DeviceChange = deviceChange
return spec.VirtualMachineConfigSpec, nil
}
func (h *Handle) Commit(ctx context.Context, sess *session.Session, waitTime *int32) error {
if h.committed {
return nil // already committed
}
// make sure there is a spec
h.SetSpec(nil)
cfg := make(map[string]string)
extraconfig.Encode(extraconfig.MapSink(cfg), h.ExecConfig)
s := h.Spec.Spec()
s.ExtraConfig = append(s.ExtraConfig, vmomi.OptionValueFromMap(cfg)...)
if err := h.Container.Commit(ctx, sess, h, waitTime); err != nil {
return err
}
h.committed = true
removeHandle(h.key)
return nil
}
func TestToExtraConfig(t *testing.T) {
exec := executor.ExecutorConfig{
Common: executor.Common{
ID: "deadbeef",
Name: "configtest",
},
Sessions: map[string]*executor.SessionConfig{
"deadbeef": &executor.SessionConfig{
Cmd: executor.Cmd{
Path: "/bin/bash",
Args: []string{"/bin/bash", "-c", "echo hello"},
Dir: "/",
Env: []string{"HOME=/", "PATH=/bin"},
},
},
"beefed": &executor.SessionConfig{
Cmd: executor.Cmd{
Path: "/bin/bash",
Args: []string{"/bin/bash", "-c", "echo goodbye"},
Dir: "/",
Env: []string{"HOME=/", "PATH=/bin"},
},
},
},
Networks: map[string]*executor.NetworkEndpoint{
"eth0": &executor.NetworkEndpoint{
Static: true,
IP: &net.IPNet{IP: localhost, Mask: lmask.Mask},
Network: executor.ContainerNetwork{
Common: executor.Common{
Name: "notsure",
},
Gateway: net.IPNet{IP: gateway, Mask: gmask.Mask},
Nameservers: []net.IP{},
Pools: []ip.Range{},
Aliases: []string{},
},
},
},
}
// encode exec package's ExecutorConfig
encoded := map[string]string{}
extraconfig.Encode(extraconfig.MapSink(encoded), exec)
// decode into this package's ExecutorConfig
var decoded ExecutorConfig
extraconfig.Decode(extraconfig.MapSource(encoded), &decoded)
// the source and destination structs are different - we're doing a sparse comparison
expectedNet := exec.Networks["eth0"]
actualNet := decoded.Networks["eth0"]
assert.Equal(t, expectedNet.Common, actualNet.Common)
assert.Equal(t, expectedNet.Static, actualNet.Static)
assert.Equal(t, expectedNet.Assigned, actualNet.Assigned)
assert.Equal(t, expectedNet.Network, actualNet.Network)
expectedSession := exec.Sessions["deadbeef"]
actualSession := decoded.Sessions["deadbeef"]
assert.Equal(t, expectedSession.Cmd.Path, actualSession.Cmd.Path)
assert.Equal(t, expectedSession.Cmd.Args, actualSession.Cmd.Args)
assert.Equal(t, expectedSession.Cmd.Dir, actualSession.Cmd.Dir)
assert.Equal(t, expectedSession.Cmd.Env, actualSession.Cmd.Env)
}
// NewVirtualMachineConfigSpec returns a VirtualMachineConfigSpec
func NewVirtualMachineConfigSpec(ctx context.Context, session *session.Session, config *VirtualMachineConfigSpecConfig) (*VirtualMachineConfigSpec, error) {
defer trace.End(trace.Begin(config.ID))
log.Debugf("Adding metadata to the configspec: %+v", config.Metadata)
// TEMPORARY
// set VM name to prettyname-ID, to make it readable a little bit
// if prettyname-ID is longer than max vm name length, truncate pretty name, instead of UUID, to make it unique
nameMaxLen := maxVMNameLength - len(config.ID)
prettyName := config.Name
if len(prettyName) > nameMaxLen-1 {
prettyName = prettyName[:nameMaxLen-1]
}
fullName := fmt.Sprintf("%s-%s", prettyName, config.ID)
config.VMFullName = fullName
VMPathName := config.VMPathName
if !session.IsVSAN(ctx) {
// VMFS requires the full path to vmx or everything but the datastore is ignored
VMPathName = fmt.Sprintf("%s/%s/%s.vmx", config.VMPathName, config.VMFullName, config.ID)
}
s := &types.VirtualMachineConfigSpec{
Name: fullName,
Uuid: config.BiosUUID,
Files: &types.VirtualMachineFileInfo{
VmPathName: VMPathName,
},
NumCPUs: config.NumCPUs,
CpuHotAddEnabled: &config.VMForkEnabled, // this disables vNUMA when true
MemoryMB: config.MemoryMB,
MemoryHotAddEnabled: &config.VMForkEnabled,
ExtraConfig: []types.BaseOptionValue{
// lets us see the UUID for the containerfs disk (hidden from daemon)
&types.OptionValue{Key: "disk.EnableUUID", Value: "true"},
// needed to avoid the questions that occur when attaching multiple disks with the same uuid (bugzilla 1362918)
&types.OptionValue{Key: "answer.msg.disk.duplicateUUID", Value: "Yes"},
// needed to avoid the question that occur when opening a file backed serial port
&types.OptionValue{Key: "answer.msg.serial.file.open", Value: "Append"},
&types.OptionValue{Key: "sched.mem.lpage.maxSharedPages", Value: "256"},
// seems to be needed to avoid children hanging shortly after fork
&types.OptionValue{Key: "vmotion.checkpointSVGAPrimarySize", Value: "4194304"},
// trying this out - if it works then we need to determine if we can rely on serial0 being the correct index.
&types.OptionValue{Key: "serial0.hardwareFlowControl", Value: "TRUE"},
// https://enatai-jira.eng.vmware.com/browse/BON-257
// Hotadd memory above 3 GB not working
&types.OptionValue{Key: "memory.noHotAddOver4GB", Value: "FALSE"},
&types.OptionValue{Key: "memory.maxGrow", Value: "512"},
// http://kb.vmware.com/selfservice/microsites/search.do?language=en_US&cmd=displayKC&externalId=2030189
&types.OptionValue{Key: "tools.remindInstall", Value: "FALSE"},
&types.OptionValue{Key: "tools.upgrade.policy", Value: "manual"},
},
}
// encode the config as optionvalues
cfg := map[string]string{}
extraconfig.Encode(extraconfig.MapSink(cfg), config.Metadata)
metaCfg := vmomi.OptionValueFromMap(cfg)
// merge it with the sec
s.ExtraConfig = append(s.ExtraConfig, metaCfg...)
vmcs := &VirtualMachineConfigSpec{
Session: session,
VirtualMachineConfigSpec: s,
config: config,
}
log.Debugf("Virtual machine config spec created: %+v", vmcs)
return vmcs, nil
}
// NewVirtualMachineConfigSpec returns a VirtualMachineConfigSpec
func NewVirtualMachineConfigSpec(ctx context.Context, session *session.Session, config *VirtualMachineConfigSpecConfig) (*VirtualMachineConfigSpec, error) {
defer trace.End(trace.Begin(config.ID))
VMPathName := config.VMPathName
if !session.IsVSAN(ctx) {
// VMFS requires the full path to vmx or everything but the datastore is ignored
VMPathName = fmt.Sprintf("%s/%s/%[2]s.vmx", config.VMPathName, config.ID)
}
log.Debugf("Adding metadata to the configspec: %+v", config.Metadata)
// TEMPORARY
s := &types.VirtualMachineConfigSpec{
Name: config.ID,
Files: &types.VirtualMachineFileInfo{
VmPathName: VMPathName,
},
NumCPUs: config.NumCPUs,
CpuHotAddEnabled: &config.VMForkEnabled, // this disables vNUMA when true
MemoryMB: config.MemoryMB,
MemoryHotAddEnabled: &config.VMForkEnabled,
// needed to cause the disk uuid to propogate into linux for presentation via /dev/disk/by-id/
ExtraConfig: []types.BaseOptionValue{
// lets us see the UUID for the containerfs disk (hidden from daemon)
&types.OptionValue{Key: "disk.EnableUUID", Value: "true"},
// needed to avoid the questions that occur when attaching multiple disks with the same uuid (bugzilla 1362918)
&types.OptionValue{Key: "answer.msg.disk.duplicateUUID", Value: "Yes"},
&types.OptionValue{Key: "answer.msg.serial.file.open", Value: "Replace"},
&types.OptionValue{Key: "sched.mem.lpage.maxSharedPages", Value: "256"},
// seems to be needed to avoid children hanging shortly after fork
&types.OptionValue{Key: "vmotion.checkpointSVGAPrimarySize", Value: "4194304"},
// trying this out - if it works then we need to determine if we can rely on serial0 being the correct index.
&types.OptionValue{Key: "serial0.hardwareFlowControl", Value: "TRUE"},
// https://enatai-jira.eng.vmware.com/browse/BON-257
&types.OptionValue{Key: "memory.noHotAddOver4GB", Value: "FALSE"},
&types.OptionValue{Key: "memory.maxGrow", Value: "512"},
// http://kb.vmware.com/selfservice/microsites/search.do?language=en_US&cmd=displayKC&externalId=2030189
&types.OptionValue{Key: "tools.remindInstall", Value: "FALSE"},
&types.OptionValue{Key: "tools.upgrade.policy", Value: "manual"},
},
}
// encode the config as optionvalues
cfg := map[string]string{}
extraconfig.Encode(extraconfig.MapSink(cfg), config.Metadata)
metaCfg := extraconfig.OptionValueFromMap(cfg)
// merge it with the sec
s.ExtraConfig = append(s.ExtraConfig, metaCfg...)
return &VirtualMachineConfigSpec{
Session: session,
VirtualMachineConfigSpec: s,
config: config,
}, nil
}